From 0102500569f702cef69a964504582b82fb51d5dd Mon Sep 17 00:00:00 2001 From: Oliver Heilwagen Date: Fri, 24 Oct 2025 13:32:35 +0200 Subject: [PATCH 1/2] Add module request binary config loader for baselibs. --- .../_assets/flatbuffers_dev_mode.drawio.svg | 520 +++++++++++++++ .../_assets/flatbuffers_overview.drawio.svg | 615 ++++++++++++++++++ .../binary_config_loader/docs/index.rst | 163 +++++ 3 files changed, 1298 insertions(+) create mode 100644 docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg create mode 100644 docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg create mode 100644 docs/modules/baselibs/binary_config_loader/docs/index.rst diff --git a/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg b/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg new file mode 100644 index 00000000000..861f8cb40cc --- /dev/null +++ b/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg @@ -0,0 +1,520 @@ + + + + + + + + + + +
+
+
+ + Target + +
+
+
+ + + Target + + + + + + + + + + + +
+
+
+ score_module +
+
+
+ + + score_module + + + + + + + + + + + +
+
+
+ score::baselibs +
+
+
+ + + score::baselibs + + + + + + + + + + + + + + + +
+
+
+ load +
+
+
+ + + load + + + + + + + + + + + + + + + +
+
+
+ Create binary + + via + +
+ flatbuffers:: +
+
+ Parser +
+
+
+
+ + + Create bina... + + + + + + + + + + + + +
+
+
+ read +
+
+
+ + + read + + + + + + + + + + + + + + + +
+
+
+ access structs +
+
+
+ + + access stru... + + + + + + + + + + + +
+
+
+ flatbuffers +
+
+
+ + + flatbuffers + + + + + + + + + + + +
+
+
+ header-only/ +
+ crate/... +
+
+
+
+ + + header-only/... + + + + + + + + + + + + +
+
+
+ use +
+
+
+ + + use + + + + + + + + + + + +
+
+
+ User written +
+ code +
+
+
+
+ + + User writte... + + + + + + + + + + + + +
+
+
+ generate for +
+ + C++/Rust/ + +
+ Python/Java/... +
+
+
+
+
+ + + generate for... + + + + + + + + + + + + +
+
+
+ @flatbuffers//:flatc +
+
+
+ + + @flatbu... + + + + + + + + + + + + + + + + +
+
+
+
+ + config.fbs +
+ +
+
+ + (schema) + +
+
+
+
+ + + confi... + + + + + + + + + + + + + + + + +
+
+
+
+ config.json +
+
+
+
+ + + config... + + + + + + + + + + + + + + + + +
+
+
+ deploy +
+
+
+ + + deploy + + + + + + + + + + + + + + + + +
+
+
+
+ + config.fbs + +
+
+
+
+ + + confi... + + + + + + + + + + + +
+
+
+ flatbuffers +
+
+
+ + + flatbuffers + + + + + + + + + + + + + + + + + +
+
+
+
+ + config.json + +
+
+
+
+ + + config... + + + + + + + + + + + +
+
+
+ (Optional) Development mode +
+
+
+ + + (Optional) Development mode + + + + + + + + + + Text is not SVG - cannot display + + + + diff --git a/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg b/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg new file mode 100644 index 00000000000..102b9150471 --- /dev/null +++ b/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg @@ -0,0 +1,615 @@ + + + + + + + + + + +
+
+
+ + Target + +
+
+
+ + + Target + + + + + + + + + + + +
+
+
+ score_module +
+
+
+ + + score_module + + + + + + + + + + + + + + +
+
+
+ score::baselibs +
+
+
+ + + score::baselibs + + + + + + + + + + + + + + + +
+
+
+ load +
+
+
+ + + load + + + + + + + + + + + + + +
+
+
+ config.bin +
+
+
+ + + config... + + + + + + + + + + + +
+
+
+ Load config file +
+
+
+ + + Load config... + + + + + + + + + + + + +
+
+
+ read +
+
+
+ + + read + + + + + + + + + + + + + + + +
+
+
+ access structs +
+
+
+ + + access stru... + + + + + + + + + + + +
+
+
+ flatbuffers +
+
+
+ + + flatbuffers + + + + + + + + + + + +
+
+
+ header-only/ +
+ crate/... +
+
+
+
+ + + header-only/... + + + + + + + + + + + + + + + + + + +
+
+
+ use +
+
+
+ + + use + + + + + + + + + + + +
+
+
+ User written +
+ code +
+
+
+
+ + + User writte... + + + + + + + + + + + + +
+
+
+ generate for +
+ + C++/Rust/ + +
+ Python/Java/... +
+
+
+
+
+ + + generate for... + + + + + + + + + + + + +
+
+
+ validate +
+ generate +
+
+
+
+ + + validate... + + + + + + + + + + + + +
+
+
+ @flatbuffers//:flatc +
+
+
+ + + @flatbu... + + + + + + + + + + + + + + + + +
+
+
+
+ + config.fbs + +
+
+
+ + (schema) + +
+
+
+
+ + + confi... + + + + + + + + + + + + + + + + +
+
+
+
+ config.json +
+
+
+
+ + + config... + + + + + + + + + + + + + + + + + + + +
+
+
+ deploy +
+
+
+ + + deploy + + + + + + + + + + + + + + + + +
+
+
+ Simplified C++ example +
+
+
+ + + Simplified C++ example + + + + + + + + + + + +
+
+
+
+ + const my::namespace::MyWidget GetMyWidget(const void *buf); + +
+
+ +
+ +
+
+ + struct MyWidget ... { + +
+
+ + const my::namespace::Widget *widget() const; + +
+
+ + }; + +
+
+ + ... + +
+
+ + struct Geometry... { + +
+
+ + const my::namespace::Size *size() const; + +
+
+ + }; + +
+
+ + struct Size ... { + +
+
+ + int32_t width() const; + +
+
+ + }; + +
+
+
+
+ + + const my::namespace::MyWidget GetMyWidget(const void *buf);... + + + + + + + + + + + +
+
+
+
+ + config_file = ... binary config loader ... + +
+
+ + auto my_widget = my::namspace::GetMyWidget(config_file); + +
+
+ +
+ +
+
+ + // assumes required attribute for tables in config.fbs + +
+
+ + auto width = my_widget->widget()->geometry()->size()->width(); + +
+
+
+
+ + + config_file = ... binary config loader ...... + + + + + + + + + + Text is not SVG - cannot display + + + + diff --git a/docs/modules/baselibs/binary_config_loader/docs/index.rst b/docs/modules/baselibs/binary_config_loader/docs/index.rst new file mode 100644 index 00000000000..7826723b77e --- /dev/null +++ b/docs/modules/baselibs/binary_config_loader/docs/index.rst @@ -0,0 +1,163 @@ +.. + # ******************************************************************************* + # Copyright (c) 2025 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +binary config loader +==================== + +.. document:: binary config loader + :id: doc__binary_config_loader + :status: draft + :safety: ASIL_B + :tags: component_request + + +Abstract +-------- + +This feature introduces FlatBuffers [#flatbuffers]_ binary configuration format to replace JSON-based module configuration. +FlatBuffers provides zero-copy access, schema validation, and access code generation for C++, Rust and further languages. +The approach eliminates the need for runtime parsing and accelerates module startup times. +Safety certification covers flatc tool qualification, runtime library verification and module-level testing of generated code. + +Motivation +---------- + +Module specific configuration is a cross-cutting concern that impacts system startup time and development efficiency. +For read-only configuration scenarios, runtime parsing approaches can limit startup performance in time-critical applications. + +FlatBuffers [#flatbuffers]_ binary configuration approach addresses these engineering challenges by: + - Eliminating the need for runtime parsing to meet aggressive startup time requirements + - Providing compile-time type safety through generated access code + - Reducing development effort through automated access code generation + - Ensuring schema validation at build time + +Given its cross-cutting nature, this architectural decision shall be done early in the project. + +Rationale +--------- + +Real-world experience with complex modules (e.g. diagnostics, SOME/IP) demonstrates that read-only configuration +scenarios benefit significantly from zero-copy access patterns. For these use cases, FlatBuffers is ideal as it allows +zero-copy data access. The schema-driven code generation further accelerates development by providing +type-safe access patterns, reducing both implementation effort and the potential for configuration-related runtime errors. + +Specification +------------- + +Module configuration shall use FlatBuffers [#flatbuffers]_ binary format for read-only configuration scenarios to achieve +aggressive start-up time requirements. + +.. _flatbuffers_overview: + +Implementation +^^^^^^^^^^^^^^ + +.. figure:: _assets/flatbuffers_overview.drawio.svg + :alt: FlatBuffers overview + :align: center + :width: 80% + +| FlatBuffers schema files (``config.fbs``) define configuration structure using Interface Definition Language (IDL). +| The flatc compiler generates C++ or Rust access code from these schemas (``config.fbs``). +| The flatc compiler generates cross-platform data binary from schema (``config.fbs``) and JSON (``config.json``) input. +| Runtime access operates directly on binary config data (``config.bin``) without parsing (lazy loading). +| The flatc compiler can convert binary config data (``config.bin``) back to JSON using the schema (``config.fbs``) for debugging purposes. + +Schema Evolution +"""""""""""""""" + +Backward compatibility through: + - Optional fields for new parameters + - Default values for missing fields + - Controlled field deprecation + +Schema Versioning +""""""""""""""""" + +FlatBuffers binary files do not contain embedded schema information. Schema identification requires: + - Embedded version fields in the schema root table + - File naming conventions (e.g., config_v1.2.bin) + +Build Integration +^^^^^^^^^^^^^^^^^ + +Build system integration shall provide reusable rules for: + - Binary configuration file generation from module specific schema and user-provided JSON data + - Reverse conversion from binary to JSON for debugging purposes + +Base libs +^^^^^^^^^ + +Binary config loader provides unified interface supporting: + - Binary data file loading (production, see :ref:`FlatBuffers overview `) + - Runtime binary data creation from schema and JSON, allowing on-target JSON manipulation (development mode, QM) + +Mode selection between development and production can be transparent to calling modules. + +.. image:: _assets/flatbuffers_dev_mode.drawio.svg + :alt: FlatBuffers development mode (QM) + :align: center + :width: 80% + +Backwards Compatibility +----------------------- + +Switching from JSON to FlatBuffers for module configuration is not backwards compatible. + +Security Impact +--------------- + +No change expected when compared to JSON based configuration approach. + +Safety Impact +------------- + +**Tool Qualification**: flatc compiler qualification shall be limited to binary creation use case. +Brief qualification is supplemented by module-specific validation. + +**Verification Runtime Library**: Footprint when excluding verifier/builder classes + - C++: 12 headers, ~250 LOC (incl. comments), standard library only + - Rust: 11 files, ~150 LOC (incl. comments), core/alloc only (assumes std/serialize features disabled) + +**Verification Generated Code**: Module-level verification is equivalent to handwritten access code verification. +Module testing contributes to flatc tool validation for specific schemas. +Test from configuration data (JSON) to value verification in access APIs. + +License Impact +-------------- + +None. FlatBuffers is licensed under the Apache License Version 2.0. + +How to Teach This +----------------- + +Developer adoption requires practical examples and reusable patterns. +The module template or baselibs should provide examples for reference implementations. + +Rejected Ideas +-------------- + +**Protocol Buffers**: Requires runtime parsing and memory allocation, defeating startup time objectives. + +**Custom binary formats**: Higher development and maintenance overhead compared to proven FlatBuffers ecosystem. + +Open Issues +----------- + +No open issues identified yet. + +Footnotes +--------- + +.. [#flatbuffers] https://google.github.io/flatbuffers/ From 2087bf4b414a4387a010d0f200a09686010edfbf Mon Sep 17 00:00:00 2001 From: Oliver Heilwagen Date: Thu, 26 Mar 2026 10:08:50 +0100 Subject: [PATCH 2/2] Update component request and introduce initial set of requirements. --- .../baselibs/docs/requirements/index.rst | 10 + .../_assets/flatbuffers_dev_mode.drawio.svg | 520 ------------------ .../_assets/flatbuffers_overview.drawio.svg | 239 ++++---- .../flatbuffers/docs/architecture/index.rst | 24 + .../docs/index.rst | 139 +++-- .../flatbuffers/docs/requirements/index.rst | 312 +++++++++++ docs/modules/baselibs/index.rst | 2 + 7 files changed, 556 insertions(+), 690 deletions(-) delete mode 100644 docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg rename docs/modules/baselibs/{binary_config_loader => flatbuffers}/docs/_assets/flatbuffers_overview.drawio.svg (52%) create mode 100644 docs/modules/baselibs/flatbuffers/docs/architecture/index.rst rename docs/modules/baselibs/{binary_config_loader => flatbuffers}/docs/index.rst (50%) create mode 100644 docs/modules/baselibs/flatbuffers/docs/requirements/index.rst diff --git a/docs/features/baselibs/docs/requirements/index.rst b/docs/features/baselibs/docs/requirements/index.rst index caea7d05981..fd97ffb2e4a 100644 --- a/docs/features/baselibs/docs/requirements/index.rst +++ b/docs/features/baselibs/docs/requirements/index.rst @@ -92,6 +92,16 @@ Requirements The base libraries shall provide a JSON-Library with parsing functionality. +.. feat_req:: FlatBuffers-Library + :id: feat_req__baselibs__flatbuffers_library + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: stkh_req__functional_req__base_libraries + :status: valid + + The base libraries shall provide a FlatBuffers-Library with serialization, read access, and structural verification of FlatBuffers data. + .. feat_req:: Exception-Free Development Support :id: feat_req__baselibs__result_library :reqtype: Functional diff --git a/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg b/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg deleted file mode 100644 index 861f8cb40cc..00000000000 --- a/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_dev_mode.drawio.svg +++ /dev/null @@ -1,520 +0,0 @@ - - - - - - - - - - -
-
-
- - Target - -
-
-
- - - Target - - - - - - - - - - - -
-
-
- score_module -
-
-
- - - score_module - - - - - - - - - - - -
-
-
- score::baselibs -
-
-
- - - score::baselibs - - - - - - - - - - - - - - - -
-
-
- load -
-
-
- - - load - - - - - - - - - - - - - - - -
-
-
- Create binary - - via - -
- flatbuffers:: -
-
- Parser -
-
-
-
- - - Create bina... - - - - - - - - - - - - -
-
-
- read -
-
-
- - - read - - - - - - - - - - - - - - - -
-
-
- access structs -
-
-
- - - access stru... - - - - - - - - - - - -
-
-
- flatbuffers -
-
-
- - - flatbuffers - - - - - - - - - - - -
-
-
- header-only/ -
- crate/... -
-
-
-
- - - header-only/... - - - - - - - - - - - - -
-
-
- use -
-
-
- - - use - - - - - - - - - - - -
-
-
- User written -
- code -
-
-
-
- - - User writte... - - - - - - - - - - - - -
-
-
- generate for -
- - C++/Rust/ - -
- Python/Java/... -
-
-
-
-
- - - generate for... - - - - - - - - - - - - -
-
-
- @flatbuffers//:flatc -
-
-
- - - @flatbu... - - - - - - - - - - - - - - - - -
-
-
-
- - config.fbs -
- -
-
- - (schema) - -
-
-
-
- - - confi... - - - - - - - - - - - - - - - - -
-
-
-
- config.json -
-
-
-
- - - config... - - - - - - - - - - - - - - - - -
-
-
- deploy -
-
-
- - - deploy - - - - - - - - - - - - - - - - -
-
-
-
- - config.fbs - -
-
-
-
- - - confi... - - - - - - - - - - - -
-
-
- flatbuffers -
-
-
- - - flatbuffers - - - - - - - - - - - - - - - - - -
-
-
-
- - config.json - -
-
-
-
- - - config... - - - - - - - - - - - -
-
-
- (Optional) Development mode -
-
-
- - - (Optional) Development mode - - - - - - - - - - Text is not SVG - cannot display - - - - diff --git a/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg b/docs/modules/baselibs/flatbuffers/docs/_assets/flatbuffers_overview.drawio.svg similarity index 52% rename from docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg rename to docs/modules/baselibs/flatbuffers/docs/_assets/flatbuffers_overview.drawio.svg index 102b9150471..0cd800c55e3 100644 --- a/docs/modules/baselibs/binary_config_loader/docs/_assets/flatbuffers_overview.drawio.svg +++ b/docs/modules/baselibs/flatbuffers/docs/_assets/flatbuffers_overview.drawio.svg @@ -1,14 +1,14 @@ - + - + -
+
@@ -18,20 +18,20 @@
- + Target - + -
+
score_module @@ -39,23 +39,23 @@
- + score_module - + - + -
+
score::baselibs @@ -63,24 +63,24 @@
- + score::baselibs - + - - + + -
+
load @@ -88,22 +88,22 @@
- + load - - - + + + -
+
config.bin @@ -111,42 +111,45 @@
- + config... - + -
+
- Load config file + load file +
+ and verify buffer +
- - Load config... + + load file... - - + + -
+
read @@ -154,24 +157,24 @@
- + read - - + + - + -
+
access structs @@ -179,20 +182,20 @@
- + access stru... - + -
+
flatbuffers @@ -200,20 +203,20 @@
- + flatbuffers - + -
+
header-only/ @@ -224,27 +227,27 @@
- + header-only/... - + - + - - + + -
+
use @@ -252,20 +255,20 @@
- + use - + -
+
User written @@ -276,21 +279,21 @@
- + User writte... - - + + -
+
generate for @@ -299,28 +302,28 @@ C++/Rust/
- Python/Java/... + Python/...
- + generate for... - - + + -
+
validate @@ -331,29 +334,32 @@
- + validate... - - + + -
+
@flatbuffers//:flatc +
+ (wrapped via Starlark rules) +
- + @flatbu... @@ -361,17 +367,17 @@ - + - - + + -
+
@@ -389,25 +395,25 @@
- + confi... - - + + - - + + -
+
@@ -417,28 +423,28 @@
- + config... - - + + - + - - + + -
+
deploy @@ -446,25 +452,25 @@
- + deploy - - - + + + - + -
+
Simplified C++ example @@ -472,25 +478,25 @@
- + Simplified C++ example - + -
+
- const my::namespace::MyWidget GetMyWidget(const void *buf); + const my::namespace::MyConfig GetMyConfig(const void *buf);
@@ -500,12 +506,16 @@
- struct MyWidget ... { + struct + + MyConfig + + ... {
- const my::namespace::Widget *widget() const; + const my::namespace::AdvancedSettings *settings() const;
@@ -520,28 +530,13 @@
- struct Geometry... { - -
-
- - const my::namespace::Size *size() const; - -
-
- - }; - -
-
- - struct Size ... { + struct AdvancedSettings ... {
- - int32_t width() const; - + + int32_t my_data() const; +
@@ -552,20 +547,20 @@
- - const my::namespace::MyWidget GetMyWidget(const void *buf);... + + const my::namespace::MyConfig GetMyConfig(const void *buf);... - + -
+
@@ -575,7 +570,11 @@
- auto my_widget = my::namspace::GetMyWidget(config_file); + auto my_config = my::namspace:: + + GetMyConfig + + (config_file);
@@ -590,14 +589,18 @@
- auto width = my_widget->widget()->geometry()->size()->width(); + auto data = + + my_config + + ->settings()->my_data();
- + config_file = ... binary config loader ...... diff --git a/docs/modules/baselibs/flatbuffers/docs/architecture/index.rst b/docs/modules/baselibs/flatbuffers/docs/architecture/index.rst new file mode 100644 index 00000000000..7d5ed43509d --- /dev/null +++ b/docs/modules/baselibs/flatbuffers/docs/architecture/index.rst @@ -0,0 +1,24 @@ +.. + # ******************************************************************************* + # Copyright (c) 2025 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +FlatBuffers Component Architecture +================================== + +.. To be created. Link from comp_req to comp is mandatory for score metamodel check. + +.. comp:: JSON + :id: comp__baselibs_flatbuffers + :security: YES + :safety: ASIL_B + :status: invalid diff --git a/docs/modules/baselibs/binary_config_loader/docs/index.rst b/docs/modules/baselibs/flatbuffers/docs/index.rst similarity index 50% rename from docs/modules/baselibs/binary_config_loader/docs/index.rst rename to docs/modules/baselibs/flatbuffers/docs/index.rst index 7826723b77e..438378eeffc 100644 --- a/docs/modules/baselibs/binary_config_loader/docs/index.rst +++ b/docs/modules/baselibs/flatbuffers/docs/index.rst @@ -12,31 +12,47 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -binary config loader -==================== +FlatBuffers-Library +=================== -.. document:: binary config loader - :id: doc__binary_config_loader - :status: draft +.. document:: FlatBuffers-Library + :id: doc__flatbuffers + :status: valid :safety: ASIL_B + :security: YES :tags: component_request + :realizes: wp__cmpt_request + + +.. toctree:: + :hidden: + + requirements/index.rst + architecture/index.rst Abstract --------- +======== + +This component request proposes the integration of Google FlatBuffers [#flatbuffers]_ providing +serialization, zero-copy read access, and structural verification of FlatBuffers data, as well as code +generation via the ``flatc`` compiler. + +Additionally, the library introduces FlatBuffers binary configuration format to replace JSON-based +module configuration. FlatBuffers provides zero-copy access, schema validation, and access code +generation for C++, Rust, and further languages. The approach eliminates the need for runtime +parsing and accelerates module startup times. Safety certification covers ``flatc`` tool +qualification, runtime library verification, and module-level testing of generated code. -This feature introduces FlatBuffers [#flatbuffers]_ binary configuration format to replace JSON-based module configuration. -FlatBuffers provides zero-copy access, schema validation, and access code generation for C++, Rust and further languages. -The approach eliminates the need for runtime parsing and accelerates module startup times. -Safety certification covers flatc tool qualification, runtime library verification and module-level testing of generated code. Motivation ----------- +========== -Module specific configuration is a cross-cutting concern that impacts system startup time and development efficiency. -For read-only configuration scenarios, runtime parsing approaches can limit startup performance in time-critical applications. +Module specific configuration is a cross-cutting concern that impacts system startup time and +development efficiency. For read-only configuration scenarios, runtime parsing approaches can +limit startup performance in time-critical applications. -FlatBuffers [#flatbuffers]_ binary configuration approach addresses these engineering challenges by: +The FlatBuffers binary configuration approach addresses these engineering challenges by: - Eliminating the need for runtime parsing to meet aggressive startup time requirements - Providing compile-time type safety through generated access code - Reducing development effort through automated access code generation @@ -44,24 +60,39 @@ FlatBuffers [#flatbuffers]_ binary configuration approach addresses these engine Given its cross-cutting nature, this architectural decision shall be done early in the project. + Rationale ---------- +========= + +Real-world experience with complex modules (e.g. diagnostics, SOME/IP) demonstrates that read-only +configuration scenarios benefit significantly from zero-copy access patterns. For these use cases, +FlatBuffers is ideal as it allows zero-copy data access. The schema-driven code generation further +accelerates development by providing type-safe access patterns, reducing both implementation effort +and the potential for configuration-related runtime errors. + -Real-world experience with complex modules (e.g. diagnostics, SOME/IP) demonstrates that read-only configuration -scenarios benefit significantly from zero-copy access patterns. For these use cases, FlatBuffers is ideal as it allows -zero-copy data access. The schema-driven code generation further accelerates development by providing -type-safe access patterns, reducing both implementation effort and the potential for configuration-related runtime errors. +Basic Functionality +=================== -Specification -------------- +The ``flatc`` compiler shall generate code for serializing, accessing, and verifying +FlatBuffers binary data. Code generation shall be provided for C++ and Rust at ASIL-B level +and other languages as needed at QM level (e.g. Python). -Module configuration shall use FlatBuffers [#flatbuffers]_ binary format for read-only configuration scenarios to achieve -aggressive start-up time requirements. +The FlatBuffers-Library shall provide services for serialization, zero-copy access of +FlatBuffers binary data, and structural verification of buffers. The library shall also support +loading FlatBuffers binary files and optionally provide common buffer identification and version +checking mechanisms. + +Note: FlatBuffers verification mechanism validates structural well-formedness only (e.g. offsets, vtables, +field boundaries), not payload data integrity. + +Module configuration shall use FlatBuffers binary format for read-only configuration scenarios to +achieve aggressive start-up time requirements. .. _flatbuffers_overview: Implementation -^^^^^^^^^^^^^^ +-------------- .. figure:: _assets/flatbuffers_overview.drawio.svg :alt: FlatBuffers overview @@ -69,13 +100,13 @@ Implementation :width: 80% | FlatBuffers schema files (``config.fbs``) define configuration structure using Interface Definition Language (IDL). -| The flatc compiler generates C++ or Rust access code from these schemas (``config.fbs``). -| The flatc compiler generates cross-platform data binary from schema (``config.fbs``) and JSON (``config.json``) input. +| The ``flatc`` compiler generates C++ or Rust access code from these schemas (``config.fbs``). +| The ``flatc`` compiler generates cross-platform data binary from schema (``config.fbs``) and JSON (``config.json``) input. | Runtime access operates directly on binary config data (``config.bin``) without parsing (lazy loading). -| The flatc compiler can convert binary config data (``config.bin``) back to JSON using the schema (``config.fbs``) for debugging purposes. +| The ``flatc`` compiler can convert binary config data (``config.bin``) back to JSON using the schema (``config.fbs``) for debugging purposes. Schema Evolution -"""""""""""""""" +^^^^^^^^^^^^^^^^ Backward compatibility through: - Optional fields for new parameters @@ -83,47 +114,42 @@ Backward compatibility through: - Controlled field deprecation Schema Versioning -""""""""""""""""" +^^^^^^^^^^^^^^^^^ FlatBuffers binary files do not contain embedded schema information. Schema identification requires: - Embedded version fields in the schema root table - File naming conventions (e.g., config_v1.2.bin) Build Integration -^^^^^^^^^^^^^^^^^ +----------------- Build system integration shall provide reusable rules for: - Binary configuration file generation from module specific schema and user-provided JSON data - Reverse conversion from binary to JSON for debugging purposes -Base libs -^^^^^^^^^ +Binary Config Loading +--------------------- -Binary config loader provides unified interface supporting: - - Binary data file loading (production, see :ref:`FlatBuffers overview `) - - Runtime binary data creation from schema and JSON, allowing on-target JSON manipulation (development mode, QM) +The FlatBuffers-Library provides a unified interface for binary data file loading +(see :ref:`FlatBuffers overview `). -Mode selection between development and production can be transparent to calling modules. - -.. image:: _assets/flatbuffers_dev_mode.drawio.svg - :alt: FlatBuffers development mode (QM) - :align: center - :width: 80% Backwards Compatibility ------------------------ +======================= Switching from JSON to FlatBuffers for module configuration is not backwards compatible. + Security Impact ---------------- +=============== No change expected when compared to JSON based configuration approach. + Safety Impact -------------- +============= -**Tool Qualification**: flatc compiler qualification shall be limited to binary creation use case. +**Tool Qualification**: ``flatc`` compiler qualification shall be limited to buffer serialization use case. Brief qualification is supplemented by module-specific validation. **Verification Runtime Library**: Footprint when excluding verifier/builder classes @@ -131,33 +157,42 @@ Brief qualification is supplemented by module-specific validation. - Rust: 11 files, ~150 LOC (incl. comments), core/alloc only (assumes std/serialize features disabled) **Verification Generated Code**: Module-level verification is equivalent to handwritten access code verification. -Module testing contributes to flatc tool validation for specific schemas. +Module testing contributes to ``flatc`` tool validation for specific schemas. Test from configuration data (JSON) to value verification in access APIs. + License Impact --------------- +============== None. FlatBuffers is licensed under the Apache License Version 2.0. + How to Teach This ------------------ +================= Developer adoption requires practical examples and reusable patterns. -The module template or baselibs should provide examples for reference implementations. +The FlatBuffers-Library should provide examples for reference implementations. + Rejected Ideas --------------- +============== **Protocol Buffers**: Requires runtime parsing and memory allocation, defeating startup time objectives. **Custom binary formats**: Higher development and maintenance overhead compared to proven FlatBuffers ecosystem. + Open Issues ------------ +=========== No open issues identified yet. + +Future Extensions +================= + + Footnotes ---------- +========= .. [#flatbuffers] https://google.github.io/flatbuffers/ diff --git a/docs/modules/baselibs/flatbuffers/docs/requirements/index.rst b/docs/modules/baselibs/flatbuffers/docs/requirements/index.rst new file mode 100644 index 00000000000..7eadb3a95c3 --- /dev/null +++ b/docs/modules/baselibs/flatbuffers/docs/requirements/index.rst @@ -0,0 +1,312 @@ +.. + # ******************************************************************************* + # Copyright (c) 2025 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Requirements +############ + +.. document:: FlatBuffers Requirements + :id: doc__flatbuffers_requirements + :status: draft + :safety: ASIL_B + :security: YES + :realizes: wp__requirements_comp + +FlatBuffers Tooling Requirements +================================ + +.. Clarification required how tooling requirements shall be handled, keep status invalid for now. + +.. comp_req:: FlatBuffers Code Generation for C++ + :id: comp_req__flatbuffers__codegen_cpp + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library, feat_req__baselibs__multi_language_apis + :status: invalid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library tooling shall generate code for serialization and read access of FlatBuffers data for C++. + +.. comp_req:: FlatBuffers Code Generation for Rust + :id: comp_req__flatbuffers__codegen_rust + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library, feat_req__baselibs__multi_language_apis + :status: invalid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library tooling shall generate code for serialization and read access of FlatBuffers data for Rust. + +.. comp_req:: FlatBuffers Code Generation for Python + :id: comp_req__flatbuffers__codegen_python + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: invalid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library tooling shall generate code for serialization and read access of FlatBuffers data for Python. + + .. note:: + Python code generation is nice-to-have for benchmark testing (scale configurations). + It is not intended for safety certification (meta model check requires safety level ASIL-B). + +.. comp_req:: FlatBuffers Binary Creation from JSON + :id: comp_req__flatbuffers__tooling_json_to_bin + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: invalid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library tooling shall support creation of FlatBuffers binary files from + JSON-encoded files conforming to the corresponding FlatBuffers schema. + +.. comp_req:: FlatBuffers Data Constraint Validation + :id: comp_req__flatbuffers__tooling_data_validate + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: invalid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library tooling shall provide a mechanism to validate FlatBuffers data against + schema-defined semantic constraints such as value ranges, allowed values, and required field + presence. + + .. note:: + Support JSON Schema validation of the JSON-encoded files used as input for FlatBuffers serialization. + +.. comp_req:: FlatBuffers Schema Evolution Check + :id: comp_req__flatbuffers__tooling_evolution + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: invalid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library tooling shall provide a mechanism to check whether a new version of a + FlatBuffers schema is backward compatible with a previous version. + + .. note:: + Backward compatibility in FlatBuffers requires that existing fields are not removed or + reordered, field types are not changed, and deprecated fields retain their field identifier. + Breaking these rules silently corrupts data when old binaries access buffers produced from + a new schema or vice versa. + +FlatBuffers Library Requirements +================================ + +.. comp_req:: FlatBuffers Serialization + :id: comp_req__flatbuffers__serialization + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library shall provide functionality to serialize data into the FlatBuffers binary format. + +.. comp_req:: FlatBuffers Access + :id: comp_req__flatbuffers__access + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library shall provide functionality to read FlatBuffers binary data. + + .. note:: + FlatBuffers uses a zero-copy approach where data is accessed directly from the binary buffer. + +.. comp_req:: FlatBuffers Verification + :id: comp_req__flatbuffers__verification + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library shall provide a verification mechanism to validate the structural well-formedness of a FlatBuffers buffer. + + .. note:: + Verification only validates the buffer structure (e.g. offsets, vtables, field boundaries), + not the correctness or integrity of the payload data. + +.. comp_req:: Load FlatBuffers Binary File + :id: comp_req__flatbuffers__load_binary + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library shall provide functionality to load FlatBuffers binary files from the filesystem. + +Buffer Identification and Versioning +===================================== + +.. comp_req:: Common Buffer Identification + :id: comp_req__flatbuffers__buffer_identification + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library shall provide a common opt-in buffer identification mechanism consisting + of a major version, a minor version, and a 4-character identifier. + +.. comp_req:: Common Version Check + :id: comp_req__flatbuffers__version_check + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers-Library shall provide a common opt-in version check mechanism that validates + the major version, minor version, and 4-character identifier of a FlatBuffers buffer. + +User friendly API for information exchange +========================================== + +.. comp_req:: Support for programming language idioms + :id: comp_req__flatbuffers__lang_idioms + :reqtype: Non-Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library, feat_req__baselibs__consistent_apis + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The public API shall support the idioms of the programming language it is written in. + +.. comp_req:: Use programming language infrastructure + :id: comp_req__flatbuffers__lang_infra + :reqtype: Non-Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The public API shall use core infrastructure of its programming language and accompanying standard libraries, + whenever possible and meaningful. + + .. note:: + This includes error handling. + +Full testability for the user facing API +======================================== + +.. comp_req:: Fully testable public API + :id: comp_req__flatbuffers__full_testability + :reqtype: Non-Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The public API of the library shall support dependency injection with test doubles. + + .. note:: + This enables full testability of the user code. + +Safety Impact +============= + +.. comp_req:: FlatBuffers library ASIL level + :id: comp_req__flatbuffers__asil + :reqtype: Functional + :security: NO + :safety: ASIL_B + :satisfies: feat_req__baselibs__flatbuffers_library, feat_req__baselibs__safety + :status: valid + :belongs_to: comp__baselibs_flatbuffers + + The FlatBuffers library shall be ASIL-B compliant for C++ and Rust language support. + +AoU Requirements +================= + +.. aou_req:: FlatBuffers data integrity + :id: aou_req__flatbuffers__data_integrity + :reqtype: Non-Functional + :security: NO + :safety: ASIL_B + :status: valid + + The user shall provide FlatBuffers binary data as input which is not corrupted due to HW, QM SW, or communication channel errors. + + .. note:: + The FlatBuffers-Library verification mechanism only validates structural well-formedness, + not payload data integrity. Data integrity shall be ensured by external means such as a safe + read-only filesystem for FlatBuffers binary file storage or a checksum protection on the + FlatBuffers binary file content. + +.. aou_req:: FlatBuffers access control + :id: aou_req__flatbuffers__access_control + :reqtype: Non-Functional + :security: NO + :safety: ASIL_B + :status: valid + + The user shall ensure access control and manipulation prevention on the FlatBuffers binary files. + + .. note:: + This can be done by the hosting process and system configuration (e.g. by using dm-verity). + +.. aou_req:: FlatBuffers schema consistency + :id: aou_req__flatbuffers__schema_consistency + :reqtype: Non-Functional + :security: NO + :safety: ASIL_B + :status: valid + + The user shall ensure that the FlatBuffers schema used for code generation is consistent with the + schema used to produce the binary data being read. + +.. aou_req:: FlatBuffers buffer verification before access + :id: aou_req__flatbuffers__verify_before_access + :reqtype: Functional + :security: NO + :safety: ASIL_B + :status: valid + + The user shall apply the FlatBuffers-Library verification mechanism to a loaded buffer and confirm + a successful result before accessing any data from that buffer. + + .. note:: + Accessing data from an unverified or malformed buffer results in undefined behaviour. + The verification mechanism provided by :need:`comp_req__flatbuffers__verification` only checks + structural well-formedness; it does not replace data integrity measures required by + :need:`aou_req__flatbuffers__data_integrity`. + + +.. needextend:: "__flatbuffers__" in id + :+tags: baselibs diff --git a/docs/modules/baselibs/index.rst b/docs/modules/baselibs/index.rst index c161d5abe19..b3c8052a739 100644 --- a/docs/modules/baselibs/index.rst +++ b/docs/modules/baselibs/index.rst @@ -35,6 +35,8 @@ Components and an intrusive linked list implementation conforming to the `P0406R1 proposal `_. - :need:`doc__json`: JSON abstraction layer that can switch between different parsers/serializers under the hood. +- :need:`doc__flatbuffers`: FlatBuffers-Library with serialization, read access, and structural + verification of FlatBuffers data, plus code generation via ``flatc`` for C++, Rust, and Python. - :need:`doc__filesystem`: Filesystem manipulation library similar to ``std::filesystem``. - :need:`doc__futurecpp`: Extends the C++17 Standard Library with features from newer C++ standards up to C++26, as well as selected proposals for the C++ Standard Library.