cffi: adapt bindings for libyang 5.x API changes

libyang 5.x introduced several breaking changes compared to 4.x:

- Bump minimum required soversion to 5.3.0.
- Add ly_yang_module_dir() and ly_ctx_set_options() to cdefs.h.
- In Context.__init__, always add ly_yang_module_dir() to the search path
  so that internal modules (no longer embedded in 5.x) are found at context
  creation time. When disable_searchdirs=True, apply LY_CTX_DISABLE_SEARCHDIRS
  via ly_ctx_set_options() after context creation rather than passing it to
  ly_ctx_new(), which would prevent loading of internal modules.
- ly_log_options(): int -> uint32_t.
- lyd_node.parent and related fields: lyd_node_inner * -> lyd_node *.
- Remove LYD_NEW_VAL_BIN and LYS_GETNEXT_WITHSCHEMAMOUNT (removed in 5.x).
- lys_find_child(): extended signature with ly_ctx and name filter params.
- lyd_node_any: replace lyd_any_value union and LYD_ANYDATA_VALUETYPE
  with child/children_ht/value/hints fields matching the new struct layout.
- lyd_any_value_str(): add LYD_FORMAT parameter and pass data_format(fmt)
  from the Python callers.
- lyd_dup_siblings/single(): lyd_node_inner * -> lyd_node *.
- Update tests for libyang 5.x behavioral changes: module import callbacks
  are also called for transitive imports, and error data paths may omit
  intermediate containers.

Signed-off-by: Pepa Hajek <hajekpepa@gmail.com>

Author: Pepa Hajek

cffi/cdefs.h

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: MIT
  */
 struct ly_ctx;
+struct ly_ht;
 
 #define	LY_CTX_ALL_IMPLEMENTED ...
 #define LY_CTX_REF_IMPLEMENTED ...
@@ -40,6 +41,8 @@ typedef enum {
 LY_ERR ly_ctx_new(const char *, uint16_t, struct ly_ctx **);
 void ly_ctx_destroy(struct ly_ctx *);
 int ly_ctx_set_searchdir(struct ly_ctx *, const char *);
+const char *ly_yang_module_dir(void);
+LY_ERR ly_ctx_set_options(struct ly_ctx *, uint32_t);
 
 typedef enum
 {
@@ -182,7 +185,7 @@ enum ly_stmt {
 #define LY_LOLOG ...
 #define LY_LOSTORE ...
 #define LY_LOSTORE_LAST ...
-int ly_log_options(int);
+uint32_t ly_log_options(uint32_t);
 uint32_t *ly_temp_log_options(uint32_t *);
 
 LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL);
@@ -262,7 +265,7 @@ struct lyd_node {
     uint32_t hash;
     uint32_t flags;
     const struct lysc_node *schema;
-    struct lyd_node_inner *parent;
+    struct lyd_node *parent;
     struct lyd_node *next;
     struct lyd_node *prev;
     struct lyd_meta *meta;
@@ -273,7 +276,6 @@ LY_ERR lys_set_implemented(struct lys_module *,	const char **);
 
 #define LYD_NEW_VAL_OUTPUT ...
 #define LYD_NEW_VAL_STORE_ONLY ...
-#define LYD_NEW_VAL_BIN ...
 #define LYD_NEW_VAL_CANON ...
 #define LYD_NEW_META_CLEAR_DFLT ...
 #define LYD_NEW_PATH_UPDATE ...
@@ -876,9 +878,8 @@ struct lysc_ext {
 #define LYS_GETNEXT_WITHCASE ...
 #define LYS_GETNEXT_INTONPCONT ...
 #define LYS_GETNEXT_OUTPUT ...
-#define LYS_GETNEXT_WITHSCHEMAMOUNT ...
 
-const struct lysc_node* lys_find_child(const struct lysc_node *, const struct lys_module *, const char *, size_t, uint16_t, uint32_t);
+const struct lysc_node* lys_find_child(const struct ly_ctx *, const struct lysc_node *, const struct lys_module *, const char *, uint32_t, const char *, uint32_t, uint32_t);
 const struct lysc_node* lysc_node_child(const struct lysc_node *);
 const struct lysc_node_action* lysc_node_actions(const struct lysc_node *);
 const struct lysc_node_notif* lysc_node_notifs(const struct lysc_node *);
@@ -902,7 +903,7 @@ struct lyd_node_inner {
             uint32_t hash;
             uint32_t flags;
             const struct lysc_node *schema;
-            struct lyd_node_inner *parent;
+            struct lyd_node *parent;
             struct lyd_node *next;
             struct lyd_node *prev;
             struct lyd_meta *meta;
@@ -924,7 +925,7 @@ struct lyd_node_term {
             uint32_t hash;
             uint32_t flags;
             const struct lysc_node *schema;
-            struct lyd_node_inner *parent;
+            struct lyd_node *parent;
             struct lyd_node *next;
             struct lyd_node *prev;
             struct lyd_meta *meta;
@@ -1161,39 +1162,27 @@ struct lyd_meta {
     struct lyd_value value;
 };
 
-typedef enum {
-   LYD_ANYDATA_DATATREE,
-   LYD_ANYDATA_STRING,
-   LYD_ANYDATA_XML,
-   LYD_ANYDATA_JSON
-} LYD_ANYDATA_VALUETYPE;
-
-union lyd_any_value {
-    struct lyd_node *tree;
-    const char *str;
-    const char *xml;
-    const char *json;
-};
-
 struct lyd_node_any {
     union {
         struct lyd_node node;
         struct {
             uint32_t hash;
             uint32_t flags;
             const struct lysc_node *schema;
-            struct lyd_node_inner *parent;
+            struct lyd_node *parent;
             struct lyd_node *next;
             struct lyd_node *prev;
             struct lyd_meta *meta;
             void *priv;
         };
     };
-    union lyd_any_value value;
-    LYD_ANYDATA_VALUETYPE value_type;
+    struct lyd_node *child;
+    struct ly_ht *children_ht;
+    const char *value;
+    uint32_t hints;
 };
 
-LY_ERR lyd_any_value_str(const struct lyd_node *, char **);
+LY_ERR lyd_any_value_str(const struct lyd_node *, LYD_FORMAT, char **);
 
 #define LYD_MERGE_DEFAULTS ...
 #define LYD_MERGE_DESTRUCT ...
@@ -1212,8 +1201,8 @@ LY_ERR lyd_diff_apply_all(struct lyd_node **, const struct lyd_node *);
 #define LYD_DUP_WITH_FLAGS  ...
 #define LYD_DUP_WITH_PARENTS  ...
 
-LY_ERR lyd_dup_siblings(const struct lyd_node *, struct lyd_node_inner *, uint32_t, struct lyd_node **);
-LY_ERR lyd_dup_single(const struct lyd_node *, struct lyd_node_inner *, uint32_t, struct lyd_node **);
+LY_ERR lyd_dup_siblings(const struct lyd_node *, struct lyd_node *, uint32_t, struct lyd_node **);
+LY_ERR lyd_dup_single(const struct lyd_node *, struct lyd_node *, uint32_t, struct lyd_node **);
 void lyd_free_meta_single(struct lyd_meta *);
 
 struct lysp_tpdf {
@@ -1291,7 +1280,7 @@ struct lyd_node_opaq {
             uint32_t hash;
             uint32_t flags;
             const struct lysc_node *schema;
-            struct lyd_node_inner *parent;
+            struct lyd_node *parent;
             struct lyd_node *next;
             struct lyd_node *prev;
             struct lyd_meta *meta;

cffi/source.c

@@ -6,6 +6,6 @@
 #include <libyang/libyang.h>
 #include <libyang/version.h>
 
-#if LY_VERSION_MAJOR * 10000 + LY_VERSION_MINOR * 100 + LY_VERSION_MICRO < 40202
-#error "This version of libyang bindings only works with libyang soversion 4.2.2+"
+#if LY_VERSION_MAJOR * 10000 + LY_VERSION_MINOR * 100 + LY_VERSION_MICRO < 50300
+#error "This version of libyang bindings only works with libyang soversion 5.3.0+"
 #endif

libyang/context.py

@@ -216,8 +216,6 @@ def __init__(
             return  # already initialized
 
         options = 0
-        if disable_searchdirs:
-            options |= lib.LY_CTX_DISABLE_SEARCHDIRS
         if disable_searchdir_cwd:
             options |= lib.LY_CTX_DISABLE_SEARCHDIR_CWD
         if explicit_compile:
@@ -241,6 +239,9 @@ def __init__(
         ctx = ffi.new("struct ly_ctx **")
 
         search_paths = []
+        yang_module_dir = c2str(lib.ly_yang_module_dir())
+        if yang_module_dir:
+            search_paths.append(yang_module_dir)
         if "YANGPATH" in os.environ:
             search_paths.extend(os.environ["YANGPATH"].strip(": \t\r\n'\"").split(":"))
         elif "YANG_MODPATH" in os.environ:
@@ -274,6 +275,8 @@ def __init__(
         )
         if not self.cdata:
             raise self.error("cannot create context")
+        if disable_searchdirs:
+            lib.ly_ctx_set_options(self.cdata, lib.LY_CTX_DISABLE_SEARCHDIRS)
         self.external_module_loader = ContextExternalModuleLoader(self.cdata)
 
     def compile_schema(self):

libyang/data.py

@@ -80,7 +80,6 @@ def data_format(fmt_string: str) -> int:
 def newval_flags(
     rpc_output: bool = False,
     store_only: bool = False,
-    bin_value: bool = False,
     canon_value: bool = False,
     meta_clear_default: bool = False,
     update: bool = False,
@@ -94,8 +93,6 @@ def newval_flags(
         flags |= lib.LYD_NEW_VAL_OUTPUT
     if store_only:
         flags |= lib.LYD_NEW_VAL_STORE_ONLY
-    if bin_value:
-        flags |= lib.LYD_NEW_VAL_BIN
     if canon_value:
         flags |= lib.LYD_NEW_VAL_CANON
     if meta_clear_default:
@@ -391,15 +388,13 @@ def new_path(
         opt_update: bool = False,
         opt_output: bool = False,
         opt_opaq: bool = False,
-        opt_bin_value: bool = False,
         opt_canon_value: bool = False,
         opt_store_only: bool = False,
     ):
         flags = newval_flags(
             update=opt_update,
             rpc_output=opt_output,
             opaq=opt_opaq,
-            bin_value=opt_bin_value,
             canon_value=opt_canon_value,
             store_only=opt_store_only,
         )
@@ -1174,7 +1169,7 @@ class DNotif(DContainer):
 class DAnyxml(DNode):
     def value(self, fmt: str = "xml"):
         anystr = ffi.new("char **", ffi.NULL)
-        ret = lib.lyd_any_value_str(self.cdata, anystr)
+        ret = lib.lyd_any_value_str(self.cdata, data_format(fmt), anystr)
         if ret != lib.LY_SUCCESS:
             raise self.context.error("cannot get data")
         return c2str(anystr[0])
@@ -1185,7 +1180,7 @@ def value(self, fmt: str = "xml"):
 class DAnydata(DNode):
     def value(self, fmt: str = "xml"):
         anystr = ffi.new("char **", ffi.NULL)
-        ret = lib.lyd_any_value_str(self.cdata, anystr)
+        ret = lib.lyd_any_value_str(self.cdata, data_format(fmt), anystr)
         if ret != lib.LY_SUCCESS:
             raise self.context.error("cannot get data")
         return c2str(anystr[0])

libyang/schema.py

@@ -1872,7 +1872,6 @@ def iter_children_options(
     with_case: bool = False,
     into_non_presence_container: bool = False,
     output: bool = False,
-    with_schema_mount: bool = False,
 ) -> int:
     options = 0
     if with_choice:
@@ -1885,8 +1884,6 @@ def iter_children_options(
         options |= lib.LYS_GETNEXT_INTONPCONT
     if output:
         options |= lib.LYS_GETNEXT_OUTPUT
-    if with_schema_mount:
-        options |= lib.LYS_GETNEXT_WITHSCHEMAMOUNT
     return options
 
 
@@ -1901,7 +1898,6 @@ def iter_children(
     with_case: bool = False,
     into_non_presence_container: bool = False,
     output: bool = False,
-    with_schema_mount: bool = False,
 ) -> Iterator[SNode]:
     if types is None:
         types = (
@@ -1942,7 +1938,6 @@ def _skip(node) -> bool:
         with_case=with_case,
         into_non_presence_container=into_non_presence_container,
         output=output,
-        with_schema_mount=with_schema_mount,
     )
     child = lib.lys_getnext(ffi.NULL, parent, module, options)
     while child:

tests/test_context.py

@@ -1,9 +1,11 @@
 # Copyright (c) 2018-2019 Robin Jarry
 # SPDX-License-Identifier: MIT
 
+import glob
 import os
 import unittest
 
+from _libyang import lib
 from libyang import Context, LibyangError, Module, SContainer, SLeaf, SLeafList
 from libyang.util import c2str
 
@@ -153,12 +155,23 @@ def test_ctx_disable_searchdirs(self):
                 ctx.load_module("yolo-nodetypes")
 
     def test_ctx_using_clb(self):
-        def get_module_valid_clb(mod_name, *_):
+        def get_module_valid_clb(mod_name, mod_rev, *_):
             YOLO_NODETYPES_MOD_PATH = os.path.join(YANG_DIR, "yolo/yolo-nodetypes.yang")
-            self.assertEqual(mod_name, "yolo-nodetypes")
-            with open(YOLO_NODETYPES_MOD_PATH, encoding="utf-8") as f:
-                mod_str = f.read()
-            return "yang", mod_str
+            if mod_name == "yolo-nodetypes":
+                with open(YOLO_NODETYPES_MOD_PATH, encoding="utf-8") as f:
+                    return "yang", f.read()
+            # libyang 5.x calls the callback also for imports (e.g. ietf-inet-types)
+            # since internal modules are no longer embedded; serve them from the
+            # yang module dir if available.
+            yang_dir = c2str(lib.ly_yang_module_dir())
+            if yang_dir:
+                rev_suffix = f"@{mod_rev}" if mod_rev else ""
+                for path in glob.glob(
+                    os.path.join(yang_dir, f"{mod_name}{rev_suffix}*.yang")
+                ):
+                    with open(path, encoding="utf-8") as f:
+                        return "yang", f.read()
+            return None
 
         def get_module_invalid_clb(mod_name, *_):
             return None

tests/test_data.py

@@ -285,24 +285,27 @@ def test_data_parse_config_xml_multi_error(self):
                 validate_present=True,
                 validate_multi_error=True,
             )
-        self.assertEqual(
+        self.assertIn(
+            'Invalid boolean value "abcd".',
+            str(cm.exception),
+        )
+        self.assertIn(
+            "url[proto='https']/enabled",
+            str(cm.exception),
+        )
+        self.assertIn(
+            'List instance is missing its key "host".',
             str(cm.exception),
-            'failed to parse data tree: Invalid boolean value "abcd".: '
-            "Data path: /yolo-system:conf/url[proto='https']/enabled (line 6): "
-            'List instance is missing its key "host".: '
-            "Data path: /yolo-system:conf/url[proto='https'] (line 7)",
         )
 
         first = cm.exception.errors[0]
         self.assertEqual(first.msg, 'Invalid boolean value "abcd".')
-        self.assertEqual(
-            first.data_path, "/yolo-system:conf/url[proto='https']/enabled"
-        )
+        self.assertIn("url[proto='https']/enabled", first.data_path)
         self.assertEqual(first.line, 6)
 
         second = cm.exception.errors[1]
         self.assertEqual(second.msg, 'List instance is missing its key "host".')
-        self.assertEqual(second.data_path, "/yolo-system:conf/url[proto='https']")
+        self.assertIn("url[proto='https']", second.data_path)
         self.assertEqual(second.line, 7)
 
     XML_STATE = """<state xmlns="urn:yang:yolo:system">