From 84d471c01c3095d753fc4760b1a1fbcf3ac5427e Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Thu, 10 Feb 2022 22:41:56 -0600 Subject: [PATCH 01/35] Include pybind11 in build --- robowflex_library/CMakeLists.txt | 10 ++++++++++ robowflex_library/package.xml | 1 + 2 files changed, 11 insertions(+) diff --git a/robowflex_library/CMakeLists.txt b/robowflex_library/CMakeLists.txt index d55df9ca7..ac7883aa7 100644 --- a/robowflex_library/CMakeLists.txt +++ b/robowflex_library/CMakeLists.txt @@ -1,4 +1,5 @@ cmake_minimum_required(VERSION 2.8.3) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) project(robowflex_library) set(LIBRARY_NAME ${PROJECT_NAME}) @@ -20,6 +21,8 @@ find_library(YAML yaml-cpp REQUIRED) find_package(TinyXML2 REQUIRED) find_package(Eigen3 REQUIRED) find_package(HDF5 REQUIRED COMPONENTS C CXX) +# find_package(Python COMPONENTS Interpreter Development) +find_package(pybind11 CONFIG) ## ## Catkin setup @@ -112,6 +115,13 @@ add_library(${LIBRARY_NAME} ${SOURCES}) set_target_properties(${LIBRARY_NAME} PROPERTIES VERSION ${${PROJECT_NAME}_VERSION}) target_link_libraries(${LIBRARY_NAME} ${LIBRARIES}) +## +## Python bindings +## +if(${pybind11_FOUND}) + pybind11_add_module(robowflex_library_python src/python_bindings.cpp) +endif() + ## ## Scripts ## diff --git a/robowflex_library/package.xml b/robowflex_library/package.xml index a525fdc68..4c85c67e6 100644 --- a/robowflex_library/package.xml +++ b/robowflex_library/package.xml @@ -19,6 +19,7 @@ tinyxml2 eigen hdf5 + pybind11 std_msgs sensor_msgs From f626e5cce73b8ae73c33fdc35cf65a1bad3ec78e Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Thu, 10 Feb 2022 22:42:25 -0600 Subject: [PATCH 02/35] Make DEFAULT_ADAPTERS public for Python bindings --- robowflex_library/include/robowflex_library/planning.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robowflex_library/include/robowflex_library/planning.h b/robowflex_library/include/robowflex_library/planning.h index 71b4b1867..34127dbd6 100644 --- a/robowflex_library/include/robowflex_library/planning.h +++ b/robowflex_library/include/robowflex_library/planning.h @@ -361,9 +361,9 @@ namespace robowflex std::vector getPlannerConfigs() const override; + static const std::vector DEFAULT_ADAPTERS; ///< The default planning adapters. protected: static const std::string DEFAULT_PLUGIN; ///< The default OMPL plugin. - static const std::vector DEFAULT_ADAPTERS; ///< The default planning adapters. private: std::vector configs_; ///< Planning configurations loaded from \a config_file in From 1245b235e97bab86563759de8e159dbe41fb8f17 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Thu, 10 Feb 2022 22:42:59 -0600 Subject: [PATCH 03/35] WIP: making Python bindings to enable fetch_test to work in Python --- robowflex_library/src/python_bindings.cpp | 135 ++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 robowflex_library/src/python_bindings.cpp diff --git a/robowflex_library/src/python_bindings.cpp b/robowflex_library/src/python_bindings.cpp new file mode 100644 index 000000000..889658b18 --- /dev/null +++ b/robowflex_library/src/python_bindings.cpp @@ -0,0 +1,135 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; +namespace rf = robowflex; + +PYBIND11_MODULE(robowflex_library, m) +{ + m.doc() = "Robowflex: MoveIt made easy"; + + // Bindings from util.h + py::class_(m, "RobowflexException") + .def(py::init()) + .def_property_readonly("value", &rf::Exception::getValue) + .def_property_readonly("message", &rf::Exception::getMessage) + .def("what", &rf::Exception::what); + + py::class_(m, "ROS") + .def(py::init([](int argc, std::vector &argv, const std::string &name, unsigned int threads) + { return rf::ROS(argc, argv.data(), name, threads); }), + py::arg("argc"), py::arg("argv"), py::arg("name") = "robowflex", py::arg("threads") = 1) + .def("get_args", &rf::ROS::getArgs) + .def("wait", &rf::ROS::wait); + + m.def("explode", &rf::explode); + // End bindings from util.h + + // Bindings from detail/fetch.h + py::class_(m, "FetchRobot") + .def(py::init()) + .def("initialize", &rf::FetchRobot::initialize) + .def("add_casters_URDF", &rf::FetchRobot::addCastersURDF) + .def("set_base_pose", &rf::FetchRobot::setBasePose) + .def("point_head", &rf::FetchRobot::pointHead) + .def("open_gripper", &rf::FetchRobot::openGripper) + .def("close_gripper", &rf::FetchRobot::closeGripper); + + using FOPipelinePlanner = rf::OMPL::FetchOMPLPipelinePlanner; + py::class_>(m, "FetchOMPLPipelinePlanner") + .def(py::init()) + .def("initialize", &rf::OMPL::FetchOMPLPipelinePlanner::initialize, + py::arg("settings") = rf::OMPL::Settings(), + py::arg("adapters") = rf::OMPL::OMPLPipelinePlanner::DEFAULT_ADAPTERS); + // End bindings from detail/fetch.h + + // Bindings from builder.h + py::class_(m, "MotionRequestBuilder") + .def(py::init()) + .def(py::init()) + .def(py::init()) + .def(py::init()) + .def("clone", &rf::MotionRequestBuilder::clone) + .def("initialize", &rf::MotionRequestBuilder::initialize) + .def("set_planning_group", &rf::MotionRequestBuilder::setPlanningGroup) + .def("set_planner", &rf::MotionRequestBuilder::setPlanner) + .def("set_start_configuration", + static_cast &)>( + &rf::MotionRequestBuilder::setStartConfiguration)) + .def("set_start_configuration", + static_cast( + &rf::MotionRequestBuilder::setStartConfiguration)) + .def("set_start_configuration", + static_cast( + &rf::MotionRequestBuilder::setStartConfiguration)) + .def("use_scene_state_as_start", &rf::MotionRequestBuilder::useSceneStateAsStart) + .def("attach_object_to_start", &rf::MotionRequestBuilder::attachObjectToStart) + .def("attach_object_to_start_const", &rf::MotionRequestBuilder::attachObjectToStartConst) + .def("add_goal_configuration", + static_cast &)>( + &rf::MotionRequestBuilder::addGoalConfiguration)) + .def("add_goal_configuration", + static_cast( + &rf::MotionRequestBuilder::addGoalConfiguration)) + .def("add_goal_configuration", + static_cast( + &rf::MotionRequestBuilder::addGoalConfiguration)) + .def("add_goal_from_IK_query", &rf::MotionRequestBuilder::addGoalFromIKQuery) + .def("add_goal_pose", &rf::MotionRequestBuilder::addGoalPose) + .def("add_goal_region", &rf::MotionRequestBuilder::addGoalRegion) + .def("add_goal_rotary_tile", &rf::MotionRequestBuilder::addGoalRotaryTile) + .def("add_cylinder_side_grasp", &rf::MotionRequestBuilder::addCylinderSideGrasp) + .def("set_goal_configuration", + static_cast &)>( + &rf::MotionRequestBuilder::setGoalConfiguration)) + .def("set_goal_configuration", + static_cast( + &rf::MotionRequestBuilder::setGoalConfiguration)) + .def("set_goal_configuration", + static_cast( + &rf::MotionRequestBuilder::setGoalConfiguration)) + .def("set_goal_from_IK_query", &rf::MotionRequestBuilder::setGoalFromIKQuery) + .def("set_goal_pose", &rf::MotionRequestBuilder::setGoalPose) + .def("set_goal_region", &rf::MotionRequestBuilder::setGoalRegion) + .def("precompute_goal_configurations", &rf::MotionRequestBuilder::precomputeGoalConfigurations) + .def("clear_goals", &rf::MotionRequestBuilder::clearGoals) + .def("add_path_pose_constraint", &rf::MotionRequestBuilder::addPathPoseConstraint) + .def("add_path_position_constraint", &rf::MotionRequestBuilder::addPathPositionConstraint) + .def("add_path_orientation_constraint", &rf::MotionRequestBuilder::addPathOrientationConstraint) + .def("set_config", &rf::MotionRequestBuilder::setConfig) + .def("set_allowed_planning_time", &rf::MotionRequestBuilder::setAllowedPlanningTime) + .def("set_num_planning_attempts", &rf::MotionRequestBuilder::setNumPlanningAttempts) + .def("set_workspace_bounds", + static_cast( + &rf::MotionRequestBuilder::setWorkspaceBounds)) + .def("set_workspace_bounds", + static_cast &, + const Eigen::Ref &)>( + &rf::MotionRequestBuilder::setWorkspaceBounds)) + .def("swap_start_with_goal", &rf::MotionRequestBuilder::swapStartWithGoal) + .def("get_request", &rf::MotionRequestBuilder::getRequest) + .def("get_request_const", &rf::MotionRequestBuilder::getRequestConst) + .def("get_start_configuration", &rf::MotionRequestBuilder::getStartConfiguration) + .def("get_goal_configuration", &rf::MotionRequestBuilder::getGoalConfiguration) + .def("get_path_constraints", &rf::MotionRequestBuilder::getPathConstraints) + .def("get_robot", &rf::MotionRequestBuilder::getRobot) + .def("get_planner", &rf::MotionRequestBuilder::getPlanner) + .def("get_planning_group", &rf::MotionRequestBuilder::getPlanningGroup) + .def("get_planner_config", &rf::MotionRequestBuilder::getPlannerConfig) + .def("to_yaml_file", &rf::MotionRequestBuilder::toYAMLFile) + .def("from_yaml_file", &rf::MotionRequestBuilder::fromYAMLFile); + // End bindings from builder.h + + // TODO: Scene, Robot, MotionPlanResponse, Trajectory +} From 84c2eff067377ea9db815acb1212dc326ea0f82a Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sat, 12 Feb 2022 01:45:10 -0600 Subject: [PATCH 04/35] Initial binding generator script --- robowflex_library/generate_bindings.py | 315 +++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 robowflex_library/generate_bindings.py diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py new file mode 100644 index 000000000..876130129 --- /dev/null +++ b/robowflex_library/generate_bindings.py @@ -0,0 +1,315 @@ +import re +from collections import defaultdict +from pathlib import Path +from sys import argv +from typing import Dict, Iterable, List, Set, Tuple + +import clang.cindex +from clang.cindex import (AccessSpecifier, Cursor, CursorKind, Index, + TranslationUnit) + + +def load_data(compilation_database_file: Path, + cpp_file: Path) -> Tuple[Index, TranslationUnit]: + # Parse the given input file + index = clang.cindex.Index.create() + comp_db = clang.cindex.CompilationDatabase.fromDirectory( + compilation_database_file) + commands = comp_db.getCompileCommands(cpp_file) + file_args = [] + for command in commands: + for argument in command.arguments: + file_args.append(argument) + file_args = file_args[2:-1] + translation_unit = index.parse(cpp_file, file_args) + return index, translation_unit + + +def get_nodes_from_file(nodes: Iterable[Cursor], + file_name: str) -> List[Cursor]: + return list(filter(lambda n: n.location.file.name == file_name, nodes)) + + +def get_nodes_with_kind(nodes: Iterable[Cursor], + node_kinds: Iterable[CursorKind]) -> List[Cursor]: + return list(filter(lambda n: n.kind in node_kinds, nodes)) + + +# This approach to snake-case conversion adapted from: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +SPECIAL_KEYWORDS = re.compile(r'ID|YAML|SRDF|URDF|JSON|IK') +NAIVE_CAMEL_CASE = re.compile(r'(.)([A-Z][a-z]+)') +CAMEL_CASE_UNDERSCORES = re.compile(r'__([A-Z])') +ADVANCED_CAMEL_CASE = re.compile(r'([a-z0-9])([A-Z])') + + +def downcase_keywords(kwd_match: re.Match) -> str: + kwd = kwd_match.group(0) + return kwd[0] + kwd[1:].lower() + + +def to_snake_case(camel_name: str) -> str: + snake_name = SPECIAL_KEYWORDS.sub(downcase_keywords, camel_name) + snake_name = NAIVE_CAMEL_CASE.sub(r'\1_\2', snake_name) + snake_name = CAMEL_CASE_UNDERSCORES.sub(r'_\1', snake_name) + snake_name = ADVANCED_CAMEL_CASE.sub(r'\1_\2', snake_name) + return snake_name.lower() + + +def generate_function_pointer_signature(ns_name: str, + function_node: Cursor, + class_node: Cursor = None) -> str: + signature = f'{function_node.type.get_result().spelling} ({ns_name + "::" + class_node.spelling + "::*" if class_node else "*"})(' + for typ in function_node.type.argument_types(): + signature += f'{typ.spelling}, ' + + signature = signature[:-2] + ')' + return signature + + +def generate_overloads(name: str, + nodes: Iterable[Cursor], + ns_name: str, + class_node: Cursor = None) -> List[str]: + overloads = [] + for function_node in nodes: + function_pointer_signature = generate_function_pointer_signature( + ns_name, function_node, class_node) + overloads.append( + f'.def("{name}, static_cast<{function_pointer_signature}>(&{ns_name}::{class_node.spelling + "::" if class_node else ""}{function_node.spelling}))' + ) + + return overloads + + +def is_exposed(node: Cursor) -> bool: + return node.access_specifier == AccessSpecifier.PUBLIC # type: ignore + + +# TODO: This could be more efficient if we called get_children once per class and iterated over it +# multiple times, at the cost of threading that list through in the parameters +# TODO: Could also improve efficiency with stronger preference for iterators over explicit lists + + +def get_exposed_fields(class_node: Cursor) -> List[Cursor]: + fields = get_nodes_with_kind(class_node.get_children(), + [CursorKind.FIELD_DECL]) # type: ignore + return list(filter(is_exposed, fields)) + + +def get_exposed_static_variables(class_node: Cursor) -> List[Cursor]: + fields = get_nodes_with_kind(class_node.get_children(), + [CursorKind.VAR_DECL]) # type: ignore + return list(filter(is_exposed, fields)) + + +def get_exposed_methods(class_node: Cursor) -> List[Cursor]: + methods = get_nodes_with_kind(class_node.get_children(), + [CursorKind.CXX_METHOD]) # type: ignore + return list(filter(is_exposed, methods)) + + +# TODO: Handle default arguments +# TODO: Maybe generate keyword args? + + +def generate_constructors(class_node: Cursor) -> List[str]: + constructors = [] + for constructor_node in get_nodes_with_kind( + class_node.get_children(), + [CursorKind.CONSTRUCTOR]): # type: ignore + constructors.append( + f".def(py::init<{', '.join([typ.spelling for typ in constructor_node.type.argument_types()])}>())" + ) + + return constructors + + +def generate_methods(class_node: Cursor, ns_name: str) -> List[str]: + class_method_nodes = defaultdict(list) + for method_node in get_exposed_methods(class_node): + class_method_nodes[to_snake_case( + method_node.spelling)].append(method_node) + + methods = [] + for method_name, method_nodes in class_method_nodes.items(): + if len(method_nodes) > 1: + methods.extend( + generate_overloads(method_name, method_nodes, ns_name, + class_node)) + else: + methods.append( + f'.def("{method_name}", &{ns_name}::{class_node.spelling}::{method_nodes[0].spelling})' + ) + + return methods + + +def generate_fields(class_node: Cursor, ns_name: str) -> List[str]: + fields = [] + # Instance fields + for field_node in get_exposed_fields(class_node): + field_binder = '.def_readwrite' + if field_node.type.is_const_qualified(): + field_binder = '.def_readonly' + + fields.append( + f'{field_binder}("{to_snake_case(field_node.spelling)}", &{ns_name}::{class_node.spelling}::{field_node.spelling})' + ) + + # Static variables + for static_var_node in get_exposed_static_variables(class_node): + var_binder = '.def_readwrite' + if static_var_node.type.is_const_qualified(): + var_binder = '.def_readonly_static' + + fields.append( + f'{var_binder}("{to_snake_case(static_var_node.spelling)}", &{ns_name}::{class_node.spelling}::{static_var_node.spelling})' + ) + + return fields + + +def get_nested_types(class_node: Cursor) -> List[Cursor]: + return get_nodes_with_kind( + class_node.get_children(), + [CursorKind.CLASS_DECL, CursorKind.STRUCT_DECL]) # type: ignore + + +def generate_class(class_node: Cursor, + ns_name: str, + pointer_names: Set[str], + parent_class: str = None) -> List[str]: + # Handle forward declarations + class_definition_node = class_node.get_definition() + if class_definition_node is None or class_definition_node != class_node: + return [] + + parent_object = f'py_{parent_class}' if parent_class else 'm' + + superclasses = [] + for superclass_node in get_nodes_with_kind( + class_node.get_children(), + [CursorKind.CXX_BASE_SPECIFIER]): # type: ignore + superclasses.append(superclass_node.type.spelling) + + superclass_string = '' + if superclasses: + superclass_string = f', {",".join(superclasses)}' + + pointer_string = '' + pointer_type_name = f'{class_node.spelling}Ptr' + if pointer_type_name in pointer_names: + pointer_string = f', {ns_name}::{pointer_type_name}' + + class_output = [ + f'// Bindings for class {ns_name}', + f'py::class_<{ns_name}::{class_node.spelling}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' + ] + # Constructors + class_output.extend(generate_constructors(class_node)) + + # Methods + class_output.extend(generate_methods(class_node, ns_name)) + + # Fields + class_output.extend(generate_fields(class_node, ns_name)) + + # Nested types + nested_output = [] + for nested_type_node in get_nested_types(class_node): + nested_output.extend( + generate_class(nested_type_node, ns_name, class_node.spelling)) + + if nested_output: + class_output[ + 1] = f'py::class_<{ns_name}::{class_node.spelling}> py_{class_node.spelling}(m, "{class_node.spelling}")' + + class_output.append(';') + return class_output + + +def get_namespaces(top_level_node: Cursor) -> List[Cursor]: + if top_level_node.kind == CursorKind.NAMESPACE: # type: ignore + return [top_level_node] + else: + return get_nodes_with_kind(top_level_node.get_children(), + [CursorKind.NAMESPACE]) # type: ignore + + +def get_pointer_defs(top_level_node: Cursor) -> Set[str]: + typedef_nodes = get_nodes_with_kind( + top_level_node.get_children(), + [CursorKind.TYPEDEF_DECL]) # type: ignore + return { + node.spelling + for node in typedef_nodes if node.spelling[-3:].lower() == 'ptr' + } + + +def bind_classes(top_level_node: Cursor) -> List[str]: + output = [] + for ns in get_namespaces(top_level_node): + pointer_names = get_pointer_defs(ns) + for class_node in get_nodes_with_kind( + ns.get_children(), + [CursorKind.CLASS_DECL, CursorKind.STRUCT_DECL]): # type: ignore + output.extend(generate_class(class_node, ns.spelling, + pointer_names)) + return output + + +def bind_functions(top_level_node: Cursor) -> List[str]: + output = [] + for ns in get_namespaces(top_level_node): + ns_functions: Dict[str, List[Cursor]] = defaultdict(list) + # We do this in two stages to handle overloads + for function_node in get_nodes_with_kind( + ns.get_children(), + [CursorKind.FUNCTION_DECL]): # type: ignore + ns_functions[to_snake_case( + function_node.spelling)].append(function_node) + + for function_name, function_nodes in ns_functions.items(): + if len(function_nodes) > 1: + output.append('m') + output.extend( + generate_overloads(function_name, function_nodes, + ns.spelling)) + output.append(';') + else: + output.append( + f'm.def("{function_name}", &{ns.spelling}::{function_nodes[0].spelling});' + ) + return output + + +def print_tree(root: Cursor, depth: int = 0): + print(f'{"".join([" "] * depth)}{root.displayname}: {root.kind}') + for child in root.get_children(): + print_tree(child, depth + 1) + + +if __name__ == '__main__': + compdb_path = Path(argv[1]) + header_path = Path(argv[2]) + module_name = argv[3] + index, translation_unit = load_data(compdb_path, header_path) + include_path = header_path.relative_to('include') + + file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), + translation_unit.spelling) + for node in file_nodes: + print_tree(node) + + output = [ + r'#include ', + f'#include <{include_path}>', 'namespace py = pybind11', + f'PYBIND11_MODULE({module_name}, m) {{' + ] + for node in file_nodes: + output.extend(bind_classes(node)) + output.extend(bind_functions(node)) + + output.append('}') + print(output) From 4651cb45f52ee2d8b79f4720b18f682db868e08f Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sat, 12 Feb 2022 01:45:24 -0600 Subject: [PATCH 05/35] WIP adding Python library to install list --- robowflex_library/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/robowflex_library/CMakeLists.txt b/robowflex_library/CMakeLists.txt index ac7883aa7..cf40f564a 100644 --- a/robowflex_library/CMakeLists.txt +++ b/robowflex_library/CMakeLists.txt @@ -21,7 +21,6 @@ find_library(YAML yaml-cpp REQUIRED) find_package(TinyXML2 REQUIRED) find_package(Eigen3 REQUIRED) find_package(HDF5 REQUIRED COMPONENTS C CXX) -# find_package(Python COMPONENTS Interpreter Development) find_package(pybind11 CONFIG) ## @@ -120,6 +119,7 @@ target_link_libraries(${LIBRARY_NAME} ${LIBRARIES}) ## if(${pybind11_FOUND}) pybind11_add_module(robowflex_library_python src/python_bindings.cpp) + set_target_properties(robowflex_library_python PROPERTIES OUTPUT_NAME "robowflex") endif() ## @@ -164,3 +164,11 @@ add_test_script(robot_scene) install_scripts() install_library() install_directory(yaml) +if(${pybind11_FOUND}) + # We do this here because it causes errors with this old version of CMake if we do it before + # finding pybind11 + find_package(Python COMPONENTS Development) + install(TARGETS robowflex_library_python + COMPONENT python + LIBRARY DESTINATION "${Python_SITELIB}/robowflex") +endif() From 1a55353882a0e6e648510c0f4dd9b2b79e204068 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sat, 12 Feb 2022 03:29:56 -0600 Subject: [PATCH 06/35] Add proper argument handling and output to file --- robowflex_library/generate_bindings.py | 40 +++++++++++++++++++++----- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 876130129..c2fc788cf 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -1,7 +1,8 @@ +import argparse import re from collections import defaultdict +from itertools import repeat from pathlib import Path -from sys import argv from typing import Dict, Iterable, List, Set, Tuple import clang.cindex @@ -291,11 +292,32 @@ def print_tree(root: Cursor, depth: int = 0): if __name__ == '__main__': - compdb_path = Path(argv[1]) - header_path = Path(argv[2]) - module_name = argv[3] - index, translation_unit = load_data(compdb_path, header_path) - include_path = header_path.relative_to('include') + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument('-m', + '--module-name', + help = 'The name of the output Python module', + type = Path) + arg_parser.add_argument('-o', + '--output-file', + help = 'Output file path', + type = Path) + arg_parser.add_argument( + '-c', + '--compilation-database', + help = 'Directory containing the project compile_commands.json', + type = Path) + arg_parser.add_argument('headers', + metavar = 'HEADER', + help = 'Header files to generate bindings for', + nargs = '+', + type = Path) + args = arg_parser.parse_args() + index, translation_unit = load_data(args.compilation_database, + args.headers[0]) + for diagnostic in translation_unit.diagnostics: + print(diagnostic.format()) + + include_path = args.headers[0].relative_to('include') file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), translation_unit.spelling) @@ -312,4 +334,8 @@ def print_tree(root: Cursor, depth: int = 0): output.extend(bind_functions(node)) output.append('}') - print(output) + with open(args.output_file, 'w') as output_file: + output_file.writelines([ + line for pair in zip(output, repeat('\n', len(output))) + for line in pair + ]) From edea3069441948ee0eeebfef5f349a2a25fb6c97 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sat, 12 Feb 2022 03:30:07 -0600 Subject: [PATCH 07/35] Remove AST print --- robowflex_library/generate_bindings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index c2fc788cf..8a08d459b 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -321,9 +321,6 @@ def print_tree(root: Cursor, depth: int = 0): file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), translation_unit.spelling) - for node in file_nodes: - print_tree(node) - output = [ r'#include ', f'#include <{include_path}>', 'namespace py = pybind11', From 5e4e628d526692c246c28b68f643debd17c9bb7d Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sat, 12 Feb 2022 03:30:32 -0600 Subject: [PATCH 08/35] Fix typos in binding templates --- robowflex_library/generate_bindings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 8a08d459b..60bd7c987 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -77,7 +77,7 @@ def generate_overloads(name: str, function_pointer_signature = generate_function_pointer_signature( ns_name, function_node, class_node) overloads.append( - f'.def("{name}, static_cast<{function_pointer_signature}>(&{ns_name}::{class_node.spelling + "::" if class_node else ""}{function_node.spelling}))' + f'.def("{name}", static_cast<{function_pointer_signature}>(&{ns_name}::{class_node.spelling + "::" if class_node else ""}{function_node.spelling}))' ) return overloads @@ -322,9 +322,9 @@ def print_tree(root: Cursor, depth: int = 0): file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), translation_unit.spelling) output = [ - r'#include ', - f'#include <{include_path}>', 'namespace py = pybind11', - f'PYBIND11_MODULE({module_name}, m) {{' + r'#include ', r'#include ', + f'#include <{include_path}>', 'namespace py = pybind11;', + f'PYBIND11_MODULE({args.module_name}, m) {{' ] for node in file_nodes: output.extend(bind_classes(node)) From 4ebebd491f5ce65b172dd34cc646a2fe88e85976 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sat, 12 Feb 2022 03:30:59 -0600 Subject: [PATCH 09/35] Fix missing stddef --- robowflex_library/generate_bindings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 60bd7c987..e771c30ae 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -21,7 +21,10 @@ def load_data(compilation_database_file: Path, for command in commands: for argument in command.arguments: file_args.append(argument) + file_args = file_args[2:-1] + # NOTE: Not sure why this needs to be manually included, but we don't find stddef otherwise + file_args.append('-I/usr/lib/clang/13.0.1/include') translation_unit = index.parse(cpp_file, file_args) return index, translation_unit From 3849d77f96c81e7ea866bddf88dfa92e643702eb Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 00:41:07 -0600 Subject: [PATCH 10/35] Operate on multiple header files at once --- robowflex_library/generate_bindings.py | 84 +++++++++++++++++--------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index e771c30ae..4f84b7165 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -7,26 +7,31 @@ import clang.cindex from clang.cindex import (AccessSpecifier, Cursor, CursorKind, Index, - TranslationUnit) + TranslationUnit, Type) def load_data(compilation_database_file: Path, - cpp_file: Path) -> Tuple[Index, TranslationUnit]: + header_paths: List[Path]) -> Tuple[Index, List[TranslationUnit]]: # Parse the given input file index = clang.cindex.Index.create() comp_db = clang.cindex.CompilationDatabase.fromDirectory( compilation_database_file) - commands = comp_db.getCompileCommands(cpp_file) - file_args = [] - for command in commands: - for argument in command.arguments: - file_args.append(argument) - - file_args = file_args[2:-1] - # NOTE: Not sure why this needs to be manually included, but we don't find stddef otherwise - file_args.append('-I/usr/lib/clang/13.0.1/include') - translation_unit = index.parse(cpp_file, file_args) - return index, translation_unit + translation_units = [] + for header_path in header_paths: + commands = comp_db.getCompileCommands(header_path) + file_args = [] + for command in commands: + for argument in command.arguments: + file_args.append(argument) + + file_args = file_args[2:-1] + # NOTE: Not sure why this needs to be manually included, but we don't find stddef otherwise + file_args.append('-I/usr/lib/clang/13.0.1/include') + translation_units.append( + index.parse(header_path, + file_args, + options = TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)) + return index, translation_units def get_nodes_from_file(nodes: Iterable[Cursor], @@ -294,6 +299,24 @@ def print_tree(root: Cursor, depth: int = 0): print_tree(child, depth + 1) +def generate_bindings(header_path: Path, + translation_unit: TranslationUnit) -> List[str]: + bindings = [] + for diagnostic in translation_unit.diagnostics: + print(diagnostic.format()) + + file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), + translation_unit.spelling) + bindings.append(f'// Bindings for {header_path}') + for node in file_nodes: + bindings.extend(bind_classes(node)) + bindings.extend(bind_functions(node)) + + bindings.append(f'// End bindings for {header_path}') + + return bindings + + if __name__ == '__main__': arg_parser = argparse.ArgumentParser() arg_parser.add_argument('-m', @@ -315,25 +338,30 @@ def print_tree(root: Cursor, depth: int = 0): nargs = '+', type = Path) args = arg_parser.parse_args() - index, translation_unit = load_data(args.compilation_database, - args.headers[0]) - for diagnostic in translation_unit.diagnostics: - print(diagnostic.format()) + prefix = [ + r'#include ', + r'#include ', + ] - include_path = args.headers[0].relative_to('include') + body = [] - file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), - translation_unit.spelling) - output = [ - r'#include ', r'#include ', - f'#include <{include_path}>', 'namespace py = pybind11;', - f'PYBIND11_MODULE({args.module_name}, m) {{' + print('Parsing header files...') + index, translation_units = load_data(args.compilation_database, + args.headers) + + print('Generating bindings...') + bodies = [ + generate_bindings(header_path, translation_unit) for header_path, + translation_unit in zip(args.headers, translation_units) ] - for node in file_nodes: - output.extend(bind_classes(node)) - output.extend(bind_functions(node)) - output.append('}') + prefix.extend(f"#include <{header_path.relative_to('include')}>" + for header_path in args.headers) + prefix.extend([ + 'namespace py = pybind11;', f'PYBIND11_MODULE({args.module_name}, m) {{' + ]) + output = prefix + [line for body in bodies for line in body] + ['}'] + print(f'Outputting bindings to {args.output_file}') with open(args.output_file, 'w') as output_file: output_file.writelines([ line for pair in zip(output, repeat('\n', len(output))) From 34aabe84b225299d4fb6066d752e4e1adc643d22 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 00:44:26 -0600 Subject: [PATCH 11/35] Handle static and const methods --- robowflex_library/generate_bindings.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 4f84b7165..5c0a76f30 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -67,12 +67,16 @@ def to_snake_case(camel_name: str) -> str: def generate_function_pointer_signature(ns_name: str, function_node: Cursor, + is_static: bool = False, class_node: Cursor = None) -> str: - signature = f'{function_node.type.get_result().spelling} ({ns_name + "::" + class_node.spelling + "::*" if class_node else "*"})(' + signature = f'{function_node.type.get_result().spelling} ({ns_name + "::" + class_node.spelling + "::*" if class_node and not is_static else "*"})(' for typ in function_node.type.argument_types(): - signature += f'{typ.spelling}, ' + signature += f'{typ.get_canonical().spelling}, ' signature = signature[:-2] + ')' + if function_node.is_const_method(): + signature += ' const' + return signature @@ -82,10 +86,11 @@ def generate_overloads(name: str, class_node: Cursor = None) -> List[str]: overloads = [] for function_node in nodes: + is_static = function_node.is_static_method() function_pointer_signature = generate_function_pointer_signature( - ns_name, function_node, class_node) + ns_name, function_node, is_static, class_node) overloads.append( - f'.def("{name}", static_cast<{function_pointer_signature}>(&{ns_name}::{class_node.spelling + "::" if class_node else ""}{function_node.spelling}))' + f'{".def_static" if is_static else ".def"}("{name}", static_cast<{function_pointer_signature}>(&{ns_name}::{class_node.spelling + "::" if class_node else ""}{function_node.spelling}))' ) return overloads @@ -148,7 +153,7 @@ def generate_methods(class_node: Cursor, ns_name: str) -> List[str]: class_node)) else: methods.append( - f'.def("{method_name}", &{ns_name}::{class_node.spelling}::{method_nodes[0].spelling})' + f'{".def_static" if method_nodes[0].is_static_method() else ".def"}("{method_name}", &{ns_name}::{class_node.spelling}::{method_nodes[0].spelling})' ) return methods From b9064faa3a5ce65be30962b7400f67a1d163316a Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 00:44:51 -0600 Subject: [PATCH 12/35] Fix syntax for nested class parent objects --- robowflex_library/generate_bindings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 5c0a76f30..6efc0f92a 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -237,7 +237,8 @@ def generate_class(class_node: Cursor, if nested_output: class_output[ - 1] = f'py::class_<{ns_name}::{class_node.spelling}> py_{class_node.spelling}(m, "{class_node.spelling}")' + 1] = f'py::class_<{ns_name}::{class_node.spelling}> py_{class_node.spelling}(m, "{class_node.spelling}");' + class_output[2] = f'py_{class_node.spelling}{class_output[2]}' class_output.append(';') return class_output From 20ac394cf449f7846c829a6738d5b9b0773632d6 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 00:45:04 -0600 Subject: [PATCH 13/35] Fix comment output for class bindings --- robowflex_library/generate_bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 6efc0f92a..2b5bebb95 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -217,7 +217,7 @@ def generate_class(class_node: Cursor, pointer_string = f', {ns_name}::{pointer_type_name}' class_output = [ - f'// Bindings for class {ns_name}', + f'// Bindings for class {ns_name}::{class_node.spelling}', f'py::class_<{ns_name}::{class_node.spelling}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' ] # Constructors From b4996ce83bd8656af4ea55c2ff6336a62428d651 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 00:45:29 -0600 Subject: [PATCH 14/35] WIP basic operator handling --- robowflex_library/generate_bindings.py | 37 +++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 2b5bebb95..1b96169da 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -96,6 +96,34 @@ def generate_overloads(name: str, return overloads +def get_base_type_name(typ: Type) -> str: + type_name_chunks = typ.spelling.split(' ') + # Check for const and reference + return ' '.join(chunk if chunk not in ['const', '&'] else '' + for chunk in type_name_chunks).strip() + + +def generate_operator_methods(operator: str, + nodes: Iterable[Cursor]) -> List[str]: + methods = [] + for method_node in nodes: + argument_type_names = [ + get_base_type_name(typ) + for typ in method_node.type.argument_types() + ] + if len(argument_type_names) >= 2: + print(method_node.spelling, argument_type_names) + + assert (len(argument_type_names) < 2) + if len(argument_type_names) == 0: + methods.append(f'.def({operator}py::self)') + else: + methods.append( + f'.def(py::self {operator} {argument_type_names[0]}())') + + return methods + + def is_exposed(node: Cursor) -> bool: return node.access_specifier == AccessSpecifier.PUBLIC # type: ignore @@ -147,7 +175,14 @@ def generate_methods(class_node: Cursor, ns_name: str) -> List[str]: methods = [] for method_name, method_nodes in class_method_nodes.items(): - if len(method_nodes) > 1: + # Handle operators + if method_name[:8] == 'operator': + operator = method_name[8:].strip() + if operator[:3] == 'new' or operator[:6] == 'delete': + continue + + methods.extend(generate_operator_methods(operator, method_nodes)) + elif len(method_nodes) > 1: methods.extend( generate_overloads(method_name, method_nodes, ns_name, class_node)) From 294a6f5d207962f7c0ecbcbc20e938a39068483f Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 00:45:42 -0600 Subject: [PATCH 15/35] Fix constructor type names --- robowflex_library/generate_bindings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 1b96169da..c8ff2db16 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -161,7 +161,7 @@ def generate_constructors(class_node: Cursor) -> List[str]: class_node.get_children(), [CursorKind.CONSTRUCTOR]): # type: ignore constructors.append( - f".def(py::init<{', '.join([typ.spelling for typ in constructor_node.type.argument_types()])}>())" + f".def(py::init<{', '.join([typ.get_canonical().spelling for typ in constructor_node.type.argument_types()])}>())" ) return constructors @@ -309,6 +309,7 @@ def bind_classes(top_level_node: Cursor) -> List[str]: return output +# TODO Templates def bind_functions(top_level_node: Cursor) -> List[str]: output = [] for ns in get_namespaces(top_level_node): From 866fbc84f2efefba850109f79c1eeddfa41bab3b Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 02:17:50 -0600 Subject: [PATCH 16/35] Use correctly qualified names for nested types --- robowflex_library/generate_bindings.py | 39 +++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index c8ff2db16..cd9d23207 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -65,11 +65,11 @@ def to_snake_case(camel_name: str) -> str: return snake_name.lower() -def generate_function_pointer_signature(ns_name: str, +def generate_function_pointer_signature(qualified_name: str, function_node: Cursor, is_static: bool = False, class_node: Cursor = None) -> str: - signature = f'{function_node.type.get_result().spelling} ({ns_name + "::" + class_node.spelling + "::*" if class_node and not is_static else "*"})(' + signature = f'{function_node.type.get_result().spelling} ({qualified_name + "::*" if class_node and not is_static else "*"})(' for typ in function_node.type.argument_types(): signature += f'{typ.get_canonical().spelling}, ' @@ -82,15 +82,15 @@ def generate_function_pointer_signature(ns_name: str, def generate_overloads(name: str, nodes: Iterable[Cursor], - ns_name: str, + qualified_name: str, class_node: Cursor = None) -> List[str]: overloads = [] for function_node in nodes: is_static = function_node.is_static_method() function_pointer_signature = generate_function_pointer_signature( - ns_name, function_node, is_static, class_node) + qualified_name, function_node, is_static, class_node) overloads.append( - f'{".def_static" if is_static else ".def"}("{name}", static_cast<{function_pointer_signature}>(&{ns_name}::{class_node.spelling + "::" if class_node else ""}{function_node.spelling}))' + f'{".def_static" if is_static else ".def"}("{name}", static_cast<{function_pointer_signature}>(&{qualified_name}::{function_node.spelling}))' ) return overloads @@ -167,7 +167,7 @@ def generate_constructors(class_node: Cursor) -> List[str]: return constructors -def generate_methods(class_node: Cursor, ns_name: str) -> List[str]: +def generate_methods(class_node: Cursor, qualified_name: str) -> List[str]: class_method_nodes = defaultdict(list) for method_node in get_exposed_methods(class_node): class_method_nodes[to_snake_case( @@ -184,17 +184,17 @@ def generate_methods(class_node: Cursor, ns_name: str) -> List[str]: methods.extend(generate_operator_methods(operator, method_nodes)) elif len(method_nodes) > 1: methods.extend( - generate_overloads(method_name, method_nodes, ns_name, + generate_overloads(method_name, method_nodes, qualified_name, class_node)) else: methods.append( - f'{".def_static" if method_nodes[0].is_static_method() else ".def"}("{method_name}", &{ns_name}::{class_node.spelling}::{method_nodes[0].spelling})' + f'{".def_static" if method_nodes[0].is_static_method() else ".def"}("{method_name}", &{qualified_name}::{method_nodes[0].spelling})' ) return methods -def generate_fields(class_node: Cursor, ns_name: str) -> List[str]: +def generate_fields(class_node: Cursor, qualified_name: str) -> List[str]: fields = [] # Instance fields for field_node in get_exposed_fields(class_node): @@ -203,7 +203,7 @@ def generate_fields(class_node: Cursor, ns_name: str) -> List[str]: field_binder = '.def_readonly' fields.append( - f'{field_binder}("{to_snake_case(field_node.spelling)}", &{ns_name}::{class_node.spelling}::{field_node.spelling})' + f'{field_binder}("{to_snake_case(field_node.spelling)}", &{qualified_name}::{field_node.spelling})' ) # Static variables @@ -213,7 +213,7 @@ def generate_fields(class_node: Cursor, ns_name: str) -> List[str]: var_binder = '.def_readonly_static' fields.append( - f'{var_binder}("{to_snake_case(static_var_node.spelling)}", &{ns_name}::{class_node.spelling}::{static_var_node.spelling})' + f'{var_binder}("{to_snake_case(static_var_node.spelling)}", &{qualified_name}::{static_var_node.spelling})' ) return fields @@ -251,31 +251,36 @@ def generate_class(class_node: Cursor, if pointer_type_name in pointer_names: pointer_string = f', {ns_name}::{pointer_type_name}' + qualified_name = f'{ns_name}::{parent_class + "::" if parent_class else ""}{class_node.spelling}' class_output = [ - f'// Bindings for class {ns_name}::{class_node.spelling}', - f'py::class_<{ns_name}::{class_node.spelling}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' + f'// Bindings for class {qualified_name}', + f'py::class_<{qualified_name}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' ] # Constructors class_output.extend(generate_constructors(class_node)) # Methods - class_output.extend(generate_methods(class_node, ns_name)) + class_output.extend(generate_methods(class_node, qualified_name)) # Fields - class_output.extend(generate_fields(class_node, ns_name)) + class_output.extend(generate_fields(class_node, qualified_name)) # Nested types nested_output = [] for nested_type_node in get_nested_types(class_node): nested_output.extend( - generate_class(nested_type_node, ns_name, class_node.spelling)) + # NOTE: If we ever have doubly-nested classes, this could be wrong - would need to pass + # qualified_name instead + generate_class(nested_type_node, ns_name, pointer_names, + class_node.spelling)) if nested_output: class_output[ - 1] = f'py::class_<{ns_name}::{class_node.spelling}> py_{class_node.spelling}(m, "{class_node.spelling}");' + 1] = f'py::class_<{qualified_name}{superclass_string}{pointer_string}> py_{class_node.spelling}({parent_object}, "{class_node.spelling}");' class_output[2] = f'py_{class_node.spelling}{class_output[2]}' class_output.append(';') + class_output.extend(nested_output) return class_output From d731d44863681d11c7395f56d3dedc4ebeea51e1 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 02:18:00 -0600 Subject: [PATCH 17/35] Special-case the assignment operator for now --- robowflex_library/generate_bindings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index cd9d23207..1b0712405 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -178,7 +178,9 @@ def generate_methods(class_node: Cursor, qualified_name: str) -> List[str]: # Handle operators if method_name[:8] == 'operator': operator = method_name[8:].strip() - if operator[:3] == 'new' or operator[:6] == 'delete': + if operator[: + 3] == 'new' or operator[: + 6] == 'delete' or operator == '=': continue methods.extend(generate_operator_methods(operator, method_nodes)) From 707e326584924ef58c57581ddfc7872d088d0672 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 02:18:19 -0600 Subject: [PATCH 18/35] Don't generate bindings for deleted constructors --- robowflex_library/generate_bindings.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 1b0712405..9884dc6e0 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -6,8 +6,8 @@ from typing import Dict, Iterable, List, Set, Tuple import clang.cindex -from clang.cindex import (AccessSpecifier, Cursor, CursorKind, Index, - TranslationUnit, Type) +from clang.cindex import (AccessSpecifier, AvailabilityKind, Cursor, CursorKind, + Index, TranslationUnit, Type) def load_data(compilation_database_file: Path, @@ -160,9 +160,10 @@ def generate_constructors(class_node: Cursor) -> List[str]: for constructor_node in get_nodes_with_kind( class_node.get_children(), [CursorKind.CONSTRUCTOR]): # type: ignore - constructors.append( - f".def(py::init<{', '.join([typ.get_canonical().spelling for typ in constructor_node.type.argument_types()])}>())" - ) + if constructor_node.availability != AvailabilityKind.NOT_AVAILABLE: # type: ignore + constructors.append( + f".def(py::init<{', '.join([typ.get_canonical().spelling for typ in constructor_node.type.argument_types()])}>())" + ) return constructors From 43c8544790632f51daca1c369099a434c24d8663 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Sun, 13 Feb 2022 02:24:51 -0600 Subject: [PATCH 19/35] Don't bind constructors for abstract classes --- robowflex_library/generate_bindings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 9884dc6e0..d74d2e11f 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -260,7 +260,8 @@ def generate_class(class_node: Cursor, f'py::class_<{qualified_name}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' ] # Constructors - class_output.extend(generate_constructors(class_node)) + if not class_node.is_abstract_record(): + class_output.extend(generate_constructors(class_node)) # Methods class_output.extend(generate_methods(class_node, qualified_name)) From fdf78167f584836d122efec8d5f9ccd68aeb9a8e Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Mon, 14 Feb 2022 14:03:18 -0600 Subject: [PATCH 20/35] Fix constructors with double-pointer arguments --- robowflex_library/generate_bindings.py | 52 +++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index d74d2e11f..2ad1b8c95 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -7,7 +7,7 @@ import clang.cindex from clang.cindex import (AccessSpecifier, AvailabilityKind, Cursor, CursorKind, - Index, TranslationUnit, Type) + Index, TranslationUnit, Type, TypeKind) def load_data(compilation_database_file: Path, @@ -153,17 +153,57 @@ def get_exposed_methods(class_node: Cursor) -> List[Cursor]: # TODO: Handle default arguments # TODO: Maybe generate keyword args? +# TODO: Maybe export constants? + + +def generate_constructor_wrapper(qualified_name: str, + argument_types: List[Type]) -> str: + '''Generate an anonymous wrapper for constructors taking double-pointer arguments.''' + # TODO: Could generalize this to wrap functions with double-pointer arguments, but I haven't found + # TODO: This whole function should be rewritten; it's messy and inefficient + # any in the codebase yet + modified_arg_types = [] + modified_arg_indices = set() + for i, typ in enumerate(argument_types): + canonical_typ = typ.get_canonical() + pointee_type = canonical_typ.get_pointee().get_pointee() + if pointee_type.kind != TypeKind.INVALID: # type: ignore + modified_arg_types.append( + f'std::vector<{canonical_typ.get_pointee().spelling}>') + modified_arg_indices.add(i) + else: + modified_arg_types.append(canonical_typ.spelling) + + arg_names = [f'arg_{i}' for i in range(len(modified_arg_types))] + lambda_args = [ + typ + ' ' + arg_name + for typ, arg_name in zip(modified_arg_types, arg_names) + ] + + invocation_expr = ', '.join( + arg_name if i not in modified_arg_indices else f'{arg_name}.data()' + for i, arg_name in enumerate(arg_names)) + return f'.def(py::init([]({", ".join(lambda_args)}) {{return {qualified_name}({invocation_expr});}}))' -def generate_constructors(class_node: Cursor) -> List[str]: + +def generate_constructors(class_node: Cursor, qualified_name: str) -> List[str]: constructors = [] for constructor_node in get_nodes_with_kind( class_node.get_children(), [CursorKind.CONSTRUCTOR]): # type: ignore if constructor_node.availability != AvailabilityKind.NOT_AVAILABLE: # type: ignore - constructors.append( - f".def(py::init<{', '.join([typ.get_canonical().spelling for typ in constructor_node.type.argument_types()])}>())" - ) + # TODO: This could be cleaner + if any(typ.get_pointee().get_pointee().kind != + TypeKind.INVALID # type: ignore + for typ in constructor_node.type.argument_types()): + constructors.append( + generate_constructor_wrapper( + qualified_name, constructor_node.type.argument_types())) + else: + constructors.append( + f".def(py::init<{', '.join([typ.get_canonical().spelling for typ in constructor_node.type.argument_types()])}>())" + ) return constructors @@ -261,7 +301,7 @@ def generate_class(class_node: Cursor, ] # Constructors if not class_node.is_abstract_record(): - class_output.extend(generate_constructors(class_node)) + class_output.extend(generate_constructors(class_node, qualified_name)) # Methods class_output.extend(generate_methods(class_node, qualified_name)) From 67cab119a8e7971ae8bb5aca0d3e03d9fec1eca7 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Mon, 14 Feb 2022 16:56:37 -0600 Subject: [PATCH 21/35] Fix output order of bindings, exception special-casing, and container of pointer special-casing --- robowflex_library/generate_bindings.py | 205 ++++++++++++++++++------- 1 file changed, 148 insertions(+), 57 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 2ad1b8c95..889c869a1 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -1,6 +1,7 @@ import argparse import re from collections import defaultdict +from dataclasses import dataclass from itertools import repeat from pathlib import Path from typing import Dict, Iterable, List, Set, Tuple @@ -156,6 +157,17 @@ def get_exposed_methods(class_node: Cursor) -> List[Cursor]: # TODO: Maybe export constants? +@dataclass +class Binding: + name: str + is_class: bool + dependencies: List[str] + body: List[str] + + def __repr__(self) -> str: + return f'Binding(name={self.name}, is_class={self.is_class}, dependencies={self.dependencies})' + + def generate_constructor_wrapper(qualified_name: str, argument_types: List[Type]) -> str: '''Generate an anonymous wrapper for constructors taking double-pointer arguments.''' @@ -168,8 +180,15 @@ def generate_constructor_wrapper(qualified_name: str, canonical_typ = typ.get_canonical() pointee_type = canonical_typ.get_pointee().get_pointee() if pointee_type.kind != TypeKind.INVALID: # type: ignore - modified_arg_types.append( - f'std::vector<{canonical_typ.get_pointee().spelling}>') + # See pybind11 known limitation #1 + if pointee_type.kind in ( + TypeKind.CHAR_U, # type: ignore + TypeKind.UCHAR): # type: ignore + vec_elem_type = 'std::string' + else: + vec_elem_type = canonical_typ.get_pointee().spelling + + modified_arg_types.append(f'std::vector<{vec_elem_type}>') modified_arg_indices.add(i) else: modified_arg_types.append(canonical_typ.spelling) @@ -181,7 +200,8 @@ def generate_constructor_wrapper(qualified_name: str, ] invocation_expr = ', '.join( - arg_name if i not in modified_arg_indices else f'{arg_name}.data()' + arg_name if i not in + modified_arg_indices else f'convertVec({arg_name}).data()' for i, arg_name in enumerate(arg_names)) return f'.def(py::init([]({", ".join(lambda_args)}) {{return {qualified_name}({invocation_expr});}}))' @@ -271,7 +291,7 @@ def get_nested_types(class_node: Cursor) -> List[Cursor]: def generate_class(class_node: Cursor, ns_name: str, pointer_names: Set[str], - parent_class: str = None) -> List[str]: + parent_class: str = None) -> List[Binding]: # Handle forward declarations class_definition_node = class_node.get_definition() if class_definition_node is None or class_definition_node != class_node: @@ -286,46 +306,62 @@ def generate_class(class_node: Cursor, superclasses.append(superclass_node.type.spelling) superclass_string = '' - if superclasses: - superclass_string = f', {",".join(superclasses)}' - pointer_string = '' pointer_type_name = f'{class_node.spelling}Ptr' if pointer_type_name in pointer_names: pointer_string = f', {ns_name}::{pointer_type_name}' qualified_name = f'{ns_name}::{parent_class + "::" if parent_class else ""}{class_node.spelling}' - class_output = [ - f'// Bindings for class {qualified_name}', - f'py::class_<{qualified_name}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' + class_output = [f'// Bindings for class {qualified_name}'] + filtered_superclasses = [ + superclass for superclass in superclasses if superclass[:5] != 'std::' ] - # Constructors - if not class_node.is_abstract_record(): - class_output.extend(generate_constructors(class_node, qualified_name)) - - # Methods - class_output.extend(generate_methods(class_node, qualified_name)) - - # Fields - class_output.extend(generate_fields(class_node, qualified_name)) - - # Nested types - nested_output = [] - for nested_type_node in get_nested_types(class_node): - nested_output.extend( - # NOTE: If we ever have doubly-nested classes, this could be wrong - would need to pass - # qualified_name instead - generate_class(nested_type_node, ns_name, pointer_names, - class_node.spelling)) - - if nested_output: - class_output[ - 1] = f'py::class_<{qualified_name}{superclass_string}{pointer_string}> py_{class_node.spelling}({parent_object}, "{class_node.spelling}");' - class_output[2] = f'py_{class_node.spelling}{class_output[2]}' + if filtered_superclasses: + superclass_string = f', {",".join(filtered_superclasses)}' + + nested_output: List[Binding] = [] + if 'std::exception' in superclasses: + class_output.append( + f'py::register_exception<{qualified_name}>({parent_object}, "{class_node.spelling}"{superclass_string})' + ) + else: + class_output.append( + f'py::class_<{qualified_name}{superclass_string}{pointer_string}>({parent_object}, "{class_node.spelling}")' + ) + # Constructors + if not class_node.is_abstract_record(): + class_output.extend( + generate_constructors(class_node, qualified_name)) + + # Methods + class_output.extend(generate_methods(class_node, qualified_name)) + + # Fields + class_output.extend(generate_fields(class_node, qualified_name)) + + # Nested types + for nested_type_node in get_nested_types(class_node): + nested_output.extend( + # NOTE: If we ever have doubly-nested classes, this could be wrong - would need to pass + # qualified_name instead + generate_class(nested_type_node, ns_name, pointer_names, + class_node.spelling)) + + if nested_output: + class_output[ + 1] = f'py::class_<{qualified_name}{superclass_string}{pointer_string}> py_{class_node.spelling}({parent_object}, "{class_node.spelling}");' + class_output[2] = f'py_{class_node.spelling}{class_output[2]}' class_output.append(';') - class_output.extend(nested_output) - return class_output + class_binding = Binding(qualified_name, + True, + dependencies = [], + body = class_output) + class_binding.dependencies.extend(filtered_superclasses) + if parent_class: + class_binding.dependencies.append(f'{ns_name}::{parent_class}') + + return [class_binding] + nested_output def get_namespaces(top_level_node: Cursor) -> List[Cursor]: @@ -346,7 +382,7 @@ def get_pointer_defs(top_level_node: Cursor) -> Set[str]: } -def bind_classes(top_level_node: Cursor) -> List[str]: +def bind_classes(top_level_node: Cursor) -> List[Binding]: output = [] for ns in get_namespaces(top_level_node): pointer_names = get_pointer_defs(ns) @@ -359,8 +395,8 @@ def bind_classes(top_level_node: Cursor) -> List[str]: # TODO Templates -def bind_functions(top_level_node: Cursor) -> List[str]: - output = [] +def bind_functions(top_level_node: Cursor) -> List[Binding]: + function_bindings = [] for ns in get_namespaces(top_level_node): ns_functions: Dict[str, List[Cursor]] = defaultdict(list) # We do this in two stages to handle overloads @@ -372,16 +408,26 @@ def bind_functions(top_level_node: Cursor) -> List[str]: for function_name, function_nodes in ns_functions.items(): if len(function_nodes) > 1: - output.append('m') - output.extend( + function_body = ['m'] + function_body.extend( generate_overloads(function_name, function_nodes, ns.spelling)) - output.append(';') + function_body.append(';') + function_bindings.append( + Binding(name = function_name, + is_class = False, + dependencies = [], + body = function_body)) else: - output.append( - f'm.def("{function_name}", &{ns.spelling}::{function_nodes[0].spelling});' - ) - return output + function_bindings.append( + Binding( + name = function_name, + is_class = False, + dependencies = [], + body = [ + f'm.def("{function_name}", &{ns.spelling}::{function_nodes[0].spelling});' + ])) + return function_bindings def print_tree(root: Cursor, depth: int = 0): @@ -390,24 +436,54 @@ def print_tree(root: Cursor, depth: int = 0): print_tree(child, depth + 1) -def generate_bindings(header_path: Path, - translation_unit: TranslationUnit) -> List[str]: +def generate_bindings(translation_unit: TranslationUnit) -> List[Binding]: bindings = [] for diagnostic in translation_unit.diagnostics: print(diagnostic.format()) file_nodes = get_nodes_from_file(translation_unit.cursor.get_children(), translation_unit.spelling) - bindings.append(f'// Bindings for {header_path}') for node in file_nodes: bindings.extend(bind_classes(node)) bindings.extend(bind_functions(node)) - bindings.append(f'// End bindings for {header_path}') - return bindings +def toposort_bindings(bindings: List[Binding]) -> List[str]: + output: List[str] = [] + class_bindings = [ + binding for binding in bindings + if binding.is_class and binding.dependencies + ] + frontier = [ + binding for binding in bindings + if binding.is_class and not binding.dependencies + ] + + while frontier: + next_binding = frontier.pop() + output.extend(next_binding.body) + new_class_bindings = [] + for binding in class_bindings: + if next_binding.name in binding.dependencies: + binding.dependencies.remove(next_binding.name) + + if not binding.dependencies: + frontier.append(binding) + else: + new_class_bindings.append(binding) + + class_bindings = new_class_bindings + + for binding in filter(lambda b: not b.is_class, bindings): + output.extend(binding.body) + + for foo in output: + assert not isinstance(foo, Binding), str(foo) + return output + + if __name__ == '__main__': arg_parser = argparse.ArgumentParser() arg_parser.add_argument('-m', @@ -430,8 +506,21 @@ def generate_bindings(header_path: Path, type = Path) args = arg_parser.parse_args() prefix = [ - r'#include ', - r'#include ', + r'#include ', r'#include ', + r''' +template +std::vector convertVec(const std::vector& vec) { + return vec; +} +template<> +std::vector convertVec(const std::vector& vec) { + std::vector result; + result.reserve(vec.size()); + for(const auto& str: vec) { + result.push_back(str.c_str()); + } + return result; +}''' ] body = [] @@ -441,17 +530,19 @@ def generate_bindings(header_path: Path, args.headers) print('Generating bindings...') - bodies = [ - generate_bindings(header_path, translation_unit) for header_path, - translation_unit in zip(args.headers, translation_units) - ] + bindings = [] + for tu in translation_units: + bindings.extend(generate_bindings(tu)) + + # Put in dependency order: + bodies = toposort_bindings(bindings) prefix.extend(f"#include <{header_path.relative_to('include')}>" for header_path in args.headers) prefix.extend([ 'namespace py = pybind11;', f'PYBIND11_MODULE({args.module_name}, m) {{' ]) - output = prefix + [line for body in bodies for line in body] + ['}'] + output = prefix + bodies + ['}'] print(f'Outputting bindings to {args.output_file}') with open(args.output_file, 'w') as output_file: output_file.writelines([ From 084bc6a1a8254ccf728e0a914e40f66d4c4cee54 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Mon, 14 Feb 2022 17:24:03 -0600 Subject: [PATCH 22/35] Hack around methods with both instance and static versions --- robowflex_library/generate_bindings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 889c869a1..ca8b08e41 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -90,8 +90,14 @@ def generate_overloads(name: str, is_static = function_node.is_static_method() function_pointer_signature = generate_function_pointer_signature( qualified_name, function_node, is_static, class_node) + if is_static: + # NOTE: This is a gross hack but seems necessary given limitations of pybind11 + overload_name = f'{name}_static' + else: + overload_name = name + overloads.append( - f'{".def_static" if is_static else ".def"}("{name}", static_cast<{function_pointer_signature}>(&{qualified_name}::{function_node.spelling}))' + f'{".def_static" if is_static else ".def"}("{overload_name}", static_cast<{function_pointer_signature}>(&{qualified_name}::{function_node.spelling}))' ) return overloads From c5e7566477f85984a400898e70202bb1c9b707ac Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Mon, 14 Feb 2022 17:24:17 -0600 Subject: [PATCH 23/35] Hack around holder types not always being defined --- robowflex_library/generate_bindings.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index ca8b08e41..265852a05 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -312,12 +312,9 @@ def generate_class(class_node: Cursor, superclasses.append(superclass_node.type.spelling) superclass_string = '' - pointer_string = '' - pointer_type_name = f'{class_node.spelling}Ptr' - if pointer_type_name in pointer_names: - pointer_string = f', {ns_name}::{pointer_type_name}' - qualified_name = f'{ns_name}::{parent_class + "::" if parent_class else ""}{class_node.spelling}' + # NOTE: We assume that everything uses shared_ptr as its holder type for simplicity + pointer_string = f', std::shared_ptr<{qualified_name}>' class_output = [f'// Bindings for class {qualified_name}'] filtered_superclasses = [ superclass for superclass in superclasses if superclass[:5] != 'std::' From a24e31ba4e819e470fdac1be4bcf25c7281c249f Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:27:55 -0600 Subject: [PATCH 24/35] Python implementation of fetch_test --- robowflex_library/fetch_test.py | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 robowflex_library/fetch_test.py diff --git a/robowflex_library/fetch_test.py b/robowflex_library/fetch_test.py new file mode 100644 index 000000000..c01f12ce3 --- /dev/null +++ b/robowflex_library/fetch_test.py @@ -0,0 +1,65 @@ +from sys import argv + +import robowflex_library as rf + +GROUP = "arm_with_torso" + +DEFAULT_ADAPTERS = [ + "default_planner_request_adapters/AddTimeParameterization", + "default_planner_request_adapters/FixWorkspaceBounds", + "default_planner_request_adapters/FixStartStateBounds", + "default_planner_request_adapters/FixStartStateCollision", + "default_planner_request_adapters/FixStartStatePathConstraints" +] + +# Startup ROS +print('Starting ROS...') +ros = rf.ROS(len(argv), argv, "robowflex", 1) +print('ROS running!') + +# Create the default Fetch robot +print('Creating Fetch robot...') +fetch = rf.FetchRobot() +fetch.initialize(True) + +# Create an empty scene +print('Creating scene...') +scene = rf.Scene(fetch) + +# Create the default planner for the Fetch +print('Creating planner...') +planner = rf.FetchOMPLPipelinePlanner(fetch, "default") +planner.initialize(rf.Settings(), DEFAULT_ADAPTERS) + +# Set the Fetch's base pose +print('Posing Fetch...') +fetch.set_base_pose(1, 1, 0.5) + +# Set the Fetch's head pose to look at a point +fetch.point_head([2, 1, 1.5]) + +# Create a motion planning request with a pose goal +print('Creating planning request...') +request = rf.MotionRequestBuilder(planner, GROUP, '') +fetch.set_group_state(GROUP, [0.05, 1.32, 1.40, -0.2, 1.72, 0.0, 1.66, 0.0]) +request.set_start_configuration(fetch.get_scratch_state()) + +# Unfurl +fetch.set_group_state(GROUP, [0.265, 0.501, 1.281, -2.272, 2.243, -2.774, 0.976, -2.007]) +request.set_goal_configuration(fetch.get_scratch_state()) +request.set_config("RRTConnect") + +# Do motion planning! +print('Motion planning...') +result = planner.plan(scene, request.get_request()) +# NOTE: We don't export the MoveIt messages library; could probably get the below constant from the +# Python bindings but I'm lazy +if result.error_code_.val != 1: + raise RuntimeError(f'Motion planning failed with code: {result.error_code_.val}') + +# Create a trajectory object for better manipulation +print('Exporting trajectory...') +trajectory = rf.Trajectory(result.trajectory_) + +# Output path to a file for visualization +trajectory.to_yaml_file("fetch_path.yml") From 18b3c06187fff303e75f4f7f9c1f5a4812bee44d Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:28:20 -0600 Subject: [PATCH 25/35] Wrap double-pointers with reference to vector --- robowflex_library/generate_bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 265852a05..6528d6fc1 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -194,7 +194,7 @@ def generate_constructor_wrapper(qualified_name: str, else: vec_elem_type = canonical_typ.get_pointee().spelling - modified_arg_types.append(f'std::vector<{vec_elem_type}>') + modified_arg_types.append(f'const std::vector<{vec_elem_type}>&') modified_arg_indices.add(i) else: modified_arg_types.append(canonical_typ.spelling) From d3cf692340506cee142057a1cd51e1245e5786ae Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:28:36 -0600 Subject: [PATCH 26/35] Constructor wrappers need to return holder types --- robowflex_library/generate_bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 6528d6fc1..9c0077313 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -210,7 +210,7 @@ def generate_constructor_wrapper(qualified_name: str, modified_arg_indices else f'convertVec({arg_name}).data()' for i, arg_name in enumerate(arg_names)) - return f'.def(py::init([]({", ".join(lambda_args)}) {{return {qualified_name}({invocation_expr});}}))' + return f'.def(py::init([]({", ".join(lambda_args)}) {{return std::make_shared<{qualified_name}>({invocation_expr});}}))' def generate_constructors(class_node: Cursor, qualified_name: str) -> List[str]: From 67ca1727ef09177ab7bfcfb1be416c906b776d70 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:29:30 -0600 Subject: [PATCH 27/35] Fix instantiation of default constructor bindings --- robowflex_library/generate_bindings.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 9c0077313..49e3cb0c8 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -227,9 +227,15 @@ def generate_constructors(class_node: Cursor, qualified_name: str) -> List[str]: generate_constructor_wrapper( qualified_name, constructor_node.type.argument_types())) else: + constructor_type_string = '' + constructor_argument_types = [ + typ.get_canonical().spelling + for typ in constructor_node.type.argument_types() + ] + if constructor_argument_types: + constructor_type_string = f'<{", ".join(constructor_argument_types)}>' constructors.append( - f".def(py::init<{', '.join([typ.get_canonical().spelling for typ in constructor_node.type.argument_types()])}>())" - ) + f".def(py::init{constructor_type_string}())") return constructors From e6eabb5b9b411b30241eee4ad77d2a3378b0a088 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:29:43 -0600 Subject: [PATCH 28/35] Only bind non-private constructors --- robowflex_library/generate_bindings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 49e3cb0c8..e5fdea1ae 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -135,6 +135,10 @@ def is_exposed(node: Cursor) -> bool: return node.access_specifier == AccessSpecifier.PUBLIC # type: ignore +def is_not_private(node: Cursor) -> bool: + return node.access_specifier != AccessSpecifier.PRIVATE # type: ignore + + # TODO: This could be more efficient if we called get_children once per class and iterated over it # multiple times, at the cost of threading that list through in the parameters # TODO: Could also improve efficiency with stronger preference for iterators over explicit lists @@ -218,7 +222,9 @@ def generate_constructors(class_node: Cursor, qualified_name: str) -> List[str]: for constructor_node in get_nodes_with_kind( class_node.get_children(), [CursorKind.CONSTRUCTOR]): # type: ignore - if constructor_node.availability != AvailabilityKind.NOT_AVAILABLE: # type: ignore + if is_not_private( + constructor_node + ) and constructor_node.availability != AvailabilityKind.NOT_AVAILABLE: # type: ignore # TODO: This could be cleaner if any(typ.get_pointee().get_pointee().kind != TypeKind.INVALID # type: ignore From 5cd3300e297d61d86d90af641dcc0b423ef7fb27 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:31:08 -0600 Subject: [PATCH 29/35] More consistently use qualified type names --- robowflex_library/generate_bindings.py | 45 ++++++++++++++++++-------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index e5fdea1ae..6d5b633a7 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -312,10 +312,14 @@ def generate_class(class_node: Cursor, parent_class: str = None) -> List[Binding]: # Handle forward declarations class_definition_node = class_node.get_definition() + qualified_name = f'{parent_class if parent_class else ns_name}::{class_node.spelling}' if class_definition_node is None or class_definition_node != class_node: + print( + f'{qualified_name} appears to be a forward declaration at {format_extent(class_node.extent)}' + ) return [] - parent_object = f'py_{parent_class}' if parent_class else 'm' + parent_object = f'py_{parent_class.replace("::", "_")}' if parent_class else 'm' superclasses = [] for superclass_node in get_nodes_with_kind( @@ -324,7 +328,8 @@ def generate_class(class_node: Cursor, superclasses.append(superclass_node.type.spelling) superclass_string = '' - qualified_name = f'{ns_name}::{parent_class + "::" if parent_class else ""}{class_node.spelling}' + print( + f'Processing {qualified_name} at {format_extent(class_node.extent)}...') # NOTE: We assume that everything uses shared_ptr as its holder type for simplicity pointer_string = f', std::shared_ptr<{qualified_name}>' class_output = [f'// Bindings for class {qualified_name}'] @@ -357,15 +362,14 @@ def generate_class(class_node: Cursor, # Nested types for nested_type_node in get_nested_types(class_node): nested_output.extend( - # NOTE: If we ever have doubly-nested classes, this could be wrong - would need to pass - # qualified_name instead generate_class(nested_type_node, ns_name, pointer_names, - class_node.spelling)) + qualified_name)) if nested_output: class_output[ - 1] = f'py::class_<{qualified_name}{superclass_string}{pointer_string}> py_{class_node.spelling}({parent_object}, "{class_node.spelling}");' - class_output[2] = f'py_{class_node.spelling}{class_output[2]}' + 1] = f'py::class_<{qualified_name}{superclass_string}{pointer_string}> py_{qualified_name.replace("::", "_")}({parent_object}, "{class_node.spelling}");' + class_output[ + 2] = f'py_{qualified_name.replace("::", "_")}{class_output[2]}' class_output.append(';') class_binding = Binding(qualified_name, @@ -374,7 +378,7 @@ def generate_class(class_node: Cursor, body = class_output) class_binding.dependencies.extend(filtered_superclasses) if parent_class: - class_binding.dependencies.append(f'{ns_name}::{parent_class}') + class_binding.dependencies.append(parent_class) return [class_binding] + nested_output @@ -400,12 +404,22 @@ def get_pointer_defs(top_level_node: Cursor) -> Set[str]: def bind_classes(top_level_node: Cursor) -> List[Binding]: output = [] for ns in get_namespaces(top_level_node): + # HACK: Only works for singly-nested namespaces + if ns.semantic_parent.kind == CursorKind.NAMESPACE: # type: ignore + qualified_name = f'{ns.semantic_parent.spelling}::{ns.spelling}' + else: + qualified_name = ns.spelling + pointer_names = get_pointer_defs(ns) for class_node in get_nodes_with_kind( ns.get_children(), - [CursorKind.CLASS_DECL, CursorKind.STRUCT_DECL]): # type: ignore - output.extend(generate_class(class_node, ns.spelling, - pointer_names)) + [ + CursorKind.CLASS_DECL, # type: ignore + CursorKind.STRUCT_DECL # type: ignore + ]): + print(class_node.spelling, class_node.access_specifier) + output.extend( + generate_class(class_node, qualified_name, pointer_names)) return output @@ -413,6 +427,11 @@ def bind_classes(top_level_node: Cursor) -> List[Binding]: def bind_functions(top_level_node: Cursor) -> List[Binding]: function_bindings = [] for ns in get_namespaces(top_level_node): + # HACK: Only works for singly-nested namespaces + if ns.semantic_parent.kind == CursorKind.NAMESPACE: # type: ignore + qualified_name = f'{ns.semantic_parent.spelling}::{ns.spelling}' + else: + qualified_name = ns.spelling ns_functions: Dict[str, List[Cursor]] = defaultdict(list) # We do this in two stages to handle overloads for function_node in get_nodes_with_kind( @@ -426,7 +445,7 @@ def bind_functions(top_level_node: Cursor) -> List[Binding]: function_body = ['m'] function_body.extend( generate_overloads(function_name, function_nodes, - ns.spelling)) + qualified_name)) function_body.append(';') function_bindings.append( Binding(name = function_name, @@ -440,7 +459,7 @@ def bind_functions(top_level_node: Cursor) -> List[Binding]: is_class = False, dependencies = [], body = [ - f'm.def("{function_name}", &{ns.spelling}::{function_nodes[0].spelling});' + f'm.def("{function_name}", &{qualified_name}::{function_nodes[0].spelling});' ])) return function_bindings From d63930f8693babbb30ddc61174f2148ae57c1dcc Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:31:23 -0600 Subject: [PATCH 30/35] Only bind non-private nested types --- robowflex_library/generate_bindings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 6d5b633a7..9b6c39308 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -360,7 +360,8 @@ def generate_class(class_node: Cursor, class_output.extend(generate_fields(class_node, qualified_name)) # Nested types - for nested_type_node in get_nested_types(class_node): + for nested_type_node in filter(is_not_private, + get_nested_types(class_node)): nested_output.extend( generate_class(nested_type_node, ns_name, pointer_names, qualified_name)) From f810d834ab2d8b85e59779f34c25f7dc8e776997 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:31:32 -0600 Subject: [PATCH 31/35] Fix handling nested namespaces --- robowflex_library/generate_bindings.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 9b6c39308..6013ad23c 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -385,11 +385,13 @@ def generate_class(class_node: Cursor, def get_namespaces(top_level_node: Cursor) -> List[Cursor]: + namespaces = [] if top_level_node.kind == CursorKind.NAMESPACE: # type: ignore - return [top_level_node] - else: - return get_nodes_with_kind(top_level_node.get_children(), - [CursorKind.NAMESPACE]) # type: ignore + namespaces.append(top_level_node) + namespaces.extend( + get_nodes_with_kind(top_level_node.get_children(), + [CursorKind.NAMESPACE])) # type: ignore + return namespaces def get_pointer_defs(top_level_node: Cursor) -> Set[str]: From 01038f8638374a453397b8cc2c1832e51576ebf9 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:31:49 -0600 Subject: [PATCH 32/35] Test that toposort worked --- robowflex_library/generate_bindings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index 6013ad23c..f208eb90e 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -516,8 +516,7 @@ def toposort_bindings(bindings: List[Binding]) -> List[str]: for binding in filter(lambda b: not b.is_class, bindings): output.extend(binding.body) - for foo in output: - assert not isinstance(foo, Binding), str(foo) + assert not class_bindings, f'Could not resolve all type dependencies! {class_bindings}' return output From bb6c2833d0b61d068c4770ef25da3d1351064344 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:32:01 -0600 Subject: [PATCH 33/35] Add missing conversion headers to prelude --- robowflex_library/generate_bindings.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index f208eb90e..c1cccd114 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -542,8 +542,13 @@ def toposort_bindings(bindings: List[Binding]) -> List[str]: type = Path) args = arg_parser.parse_args() prefix = [ - r'#include ', r'#include ', - r''' + r'''#include +#include +#include +#include +#include +#include +#include ''', r''' template std::vector convertVec(const std::vector& vec) { return vec; From 8c026b342f2dcfdc8a1e4ac3f456dc54bf2ec358 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:32:15 -0600 Subject: [PATCH 34/35] Add utility function for printing source locations --- robowflex_library/generate_bindings.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/robowflex_library/generate_bindings.py b/robowflex_library/generate_bindings.py index c1cccd114..e7201b1b6 100644 --- a/robowflex_library/generate_bindings.py +++ b/robowflex_library/generate_bindings.py @@ -8,7 +8,13 @@ import clang.cindex from clang.cindex import (AccessSpecifier, AvailabilityKind, Cursor, CursorKind, - Index, TranslationUnit, Type, TypeKind) + Index, SourceLocation, TranslationUnit, Type, + TypeKind) + + +def format_extent(extent) -> str: + start_loc: SourceLocation = extent.start + return f'{start_loc.file.name}: line {start_loc.line}' # type: ignore def load_data(compilation_database_file: Path, From d2863d65e2a1408c0ed15b4e8ac8afec7fb66170 Mon Sep 17 00:00:00 2001 From: Wil Thomason Date: Tue, 15 Feb 2022 13:32:34 -0600 Subject: [PATCH 35/35] Remove forward decl for nonexistent method --- robowflex_library/include/robowflex_library/tf.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/robowflex_library/include/robowflex_library/tf.h b/robowflex_library/include/robowflex_library/tf.h index 58bd71b68..f4094f242 100644 --- a/robowflex_library/include/robowflex_library/tf.h +++ b/robowflex_library/include/robowflex_library/tf.h @@ -169,8 +169,6 @@ namespace robowflex const RobotPose &pose, const GeometryConstPtr &geometry); - Eigen::Vector3d samplePositionConstraint(const moveit_msgs::PositionConstraint &pc); - /** \brief Get an orientation constraint message. * \param[in] ee_name The name of the end-effector link. * \param[in] base_name The frame of pose and orientation.