scripts: tdf_decoder_build: support extension TDFs

doc/custom_tdfs.md

@@ -0,0 +1,14 @@
+# Custom TDF Decoding
+
+To create a build of the tool with support for custom TDFs, the decoders need to be rebuilt
+with `tdf_decoder_build.py` being provided the custom definition file. For example:
+
+```
+./scripts/tdf_decoder_build.py --json ./scripts/tdf.json --out ./tdf/src/ --extensions ~/code/extensions/tdf.json
+```
+
+This will update the `decoders_csv.rs` and `decoders_parquet.rs` files, and the GUI and CLI
+applications can be rebuilt with a simple `cargo build --release`.
+
+> ⚠️ For MacOS builds, the resulting binaries must be notorized through Apple before
+  they can be run on other machines without warnings.

scripts/tdf_decoder_build.py

@@ -178,6 +178,15 @@ def decoders_gen(tdf_defs, output):
     for _tdf_id, info in tdf_defs["definitions"].items():
         info["arrow_schema"] = arrow_schema_expr(info, tdf_defs["structs"])
 
+    def csv_fixed_hex_bytes(field):
+        conversion = field.get("conversion", {})
+        return (
+            field["type"] == "uint8_t"
+            and field.get("num", 0) > 0
+            and conversion.get("hex", False) is True
+            and "int" not in conversion
+        )
+
     def field_conv_func(field, name_prefix=None, variable_item=False):
         t = rust_type[field["type"]]
         func = f"cursor.read_{t[0]}"
@@ -227,6 +236,10 @@ def field_conv_func(field, name_prefix=None, variable_item=False):
                         f"tdf_field_read_string_to_str(cursor, cursor_start, {field['num']}, size)?",
                     )
                 ]
+            elif csv_fixed_hex_bytes(field):
+                return [
+                    (n, f"tdf_field_read_fixed_bytes_to_hex(cursor, {field['num']})?")
+                ]
             else:
                 if field["num"] == 0:
                     if variable_item:
@@ -242,6 +255,8 @@ def field_conv_func(field, name_prefix=None, variable_item=False):
     def field_fmt(field):
         if field["type"] == "char":
             return ["{}"]
+        if csv_fixed_hex_bytes(field):
+            return ["{}"]
         if "display" in field and field["display"].get("fmt", "") == "hex":
             if digits := field["display"].get("digits", None):
                 single = [f"0x{{:0{digits}x}}"]
@@ -269,7 +284,9 @@ def csv_flatten_field(field, convs, fmt, name_prefix=None, variable_item=False):
             for struct_field in structs[field["type"]]:
                 convs += field_conv_func(
                     struct_field,
-                    name_prefix=field["name"] if name_prefix is None else f"{name_prefix}.{field['name']}",
+                    name_prefix=field["name"]
+                    if name_prefix is None
+                    else f"{name_prefix}.{field['name']}",
                     variable_item=variable_item,
                 )
             fmt += struct_fmts[field["type"]]
@@ -321,9 +338,9 @@ def csv_field_byte_size(field, repeated_item=False):
         variable_field = None
         if info["fields"]:
             last_field = info["fields"][-1]
-            if (
-                last_field.get("num", None) == 0
-                and last_field["type"] not in ("char", "uint8_t")
+            if last_field.get("num", None) == 0 and last_field["type"] not in (
+                "char",
+                "uint8_t",
             ):
                 variable_field = last_field
 
@@ -447,7 +464,9 @@ def field_byte_size(field):
             return field["num"]
         if c_type.startswith("struct "):
             struct_name = c_type.removeprefix("struct ")
-            return sum(field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"])
+            return sum(
+                field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"]
+            )
         if c_type == "char":
             return field.get("num", 0)
         base = {
@@ -473,7 +492,10 @@ def field_model(field, path):
         if c_type.startswith("struct "):
             struct_name = c_type.removeprefix("struct ")
             child_fields = tdf_defs["structs"][struct_name]["fields"]
-            children = [field_model(child, path + [arrow_name(child["name"])]) for child in child_fields]
+            children = [
+                field_model(child, path + [arrow_name(child["name"])])
+                for child in child_fields
+            ]
             if num == 0:
                 return {
                     "kind": "list",
@@ -508,16 +530,22 @@ def field_model(field, path):
             return {
                 "kind": "list",
                 "path": path,
-                "child": field_model({k: v for k, v in field.items() if k != "num"}, path),
-                "item_size": field_byte_size({k: v for k, v in field.items() if k != "num"}),
+                "child": field_model(
+                    {k: v for k, v in field.items() if k != "num"}, path
+                ),
+                "item_size": field_byte_size(
+                    {k: v for k, v in field.items() if k != "num"}
+                ),
                 "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12),
             }
 
         if num is not None and "int" not in conv:
             return {
                 "kind": "fixed_list",
                 "path": path,
-                "child": field_model({k: v for k, v in field.items() if k != "num"}, path),
+                "child": field_model(
+                    {k: v for k, v in field.items() if k != "num"}, path
+                ),
                 "num": num,
                 "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12),
             }
@@ -554,11 +582,17 @@ def model_storage_fields(model, out):
     def model_init_fields(model, out, capacity):
         kind = model["kind"]
         if kind == "primitive":
-            out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})")
+            out.append(
+                f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})"
+            )
         elif kind == "string":
-            out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})")
+            out.append(
+                f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})"
+            )
         elif kind == "binary":
-            out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})")
+            out.append(
+                f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})"
+            )
         elif kind == "fixed_list":
             model_init_fields(model["child"], out, f"{capacity} * {model['num']}")
         elif kind == "list":
@@ -647,8 +681,12 @@ def model_finish_expr(model):
                 "        }"
             )
         if kind == "struct":
-            child_fields = ",\n            ".join(child["field_expr"] for child in model["children"])
-            child_arrays = ",\n            ".join(model_finish_expr(child) for child in model["children"])
+            child_fields = ",\n            ".join(
+                child["field_expr"] for child in model["children"]
+            )
+            child_arrays = ",\n            ".join(
+                model_finish_expr(child) for child in model["children"]
+            )
             return (
                 "Arc::new(StructArray::try_new(\n"
                 f"            Fields::from(vec![\n            {child_fields}\n            ]),\n"
@@ -739,9 +777,16 @@ def write_rendered(path, template):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser("Generate rust TDF decoders", allow_abbrev=False)
     parser.add_argument("--json", required=True, type=str, help="TDF json description")
+    parser.add_argument("--extensions", type=str, help="Extension TDF json description")
     parser.add_argument("--out", required=True, type=str, help="Output folder")
     args = parser.parse_args()
 
     with open(args.json) as f:
-        definitions = json.load(f, parse_float=decimal.Decimal)
+        definitions: dict = json.load(f, parse_float=decimal.Decimal)
+    if args.extensions:
+        with open(args.extensions) as f:
+            extensions: dict = json.load(f, parse_float=decimal.Decimal)
+        definitions["structs"].update(extensions["structs"])
+        definitions["definitions"].update(extensions["definitions"])
+
     decoders_gen(definitions, args.out)

scripts/tdf_decoder_csv.rs.jinja

@@ -29,6 +29,15 @@ fn tdf_field_read_vla_to_str(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size
     Ok(format!("{}", hex::encode(buf)))
 }
 
+#[allow(dead_code)]
+fn tdf_field_read_fixed_bytes_to_hex(cursor: &mut Cursor<&[u8]>, num: usize) ->  Result<String>
+{
+    let mut buf = vec![0u8; num];
+    cursor.read_exact(&mut buf)?;
+
+    Ok(format!("{}", hex::encode(buf)))
+}
+
 fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result<usize>
 {
     if (size as usize) < base_size {
@@ -206,4 +215,15 @@ mod tests {
 
         assert_eq!(row, "0x12345678,9,abcdef");
     }
+
+    #[test]
+    fn conversion_hex_byte_array_uses_single_field_formatting() {
+        let bytes = [0xab, 0xcd, 0xef];
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let val = tdf_field_read_fixed_bytes_to_hex(&mut cursor, bytes.len()).unwrap();
+
+        assert_eq!(val, "abcdef");
+        assert_eq!(cursor.position(), bytes.len() as u64);
+    }
 }

src/main_cli.rs

@@ -45,7 +45,7 @@ impl infuse_decoder::ProgressReporter for IndicatifProgress {
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
 struct Cli {
-    /// The path to the folder containing Infuse-IoT binary files
+    /// The path to the file/folder containing Infuse-IoT binary files
     #[arg(short, long, required = true)]
     path: std::path::PathBuf,
     /// Output path for decoded files

src/main_gui.rs

@@ -1,4 +1,4 @@
-#![windows_subsystem = "windows"]
+#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
 
 use std::env;
 use std::sync::{Arc, Mutex};

tdf/src/decoders_csv.rs

@@ -227,6 +227,14 @@ fn tdf_field_read_vla_to_str(
     Ok(format!("{}", hex::encode(buf)))
 }
 
+#[allow(dead_code)]
+fn tdf_field_read_fixed_bytes_to_hex(cursor: &mut Cursor<&[u8]>, num: usize) -> Result<String> {
+    let mut buf = vec![0u8; num];
+    cursor.read_exact(&mut buf)?;
+
+    Ok(format!("{}", hex::encode(buf)))
+}
+
 fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result<usize> {
     if (size as usize) < base_size {
         return Result::Err(Error::new(
@@ -887,4 +895,15 @@ mod tests {
 
         assert_eq!(row, "0x12345678,9,abcdef");
     }
+
+    #[test]
+    fn conversion_hex_byte_array_uses_single_field_formatting() {
+        let bytes = [0xab, 0xcd, 0xef];
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let val = tdf_field_read_fixed_bytes_to_hex(&mut cursor, bytes.len()).unwrap();
+
+        assert_eq!(val, "abcdef");
+        assert_eq!(cursor.position(), bytes.len() as u64);
+    }
 }