From faab56d19b71a4a4d97d3cbc1a8484c5f207caad Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sat, 27 Jun 2026 15:17:57 +1000 Subject: [PATCH 1/5] main_gui: keep console output for debug builds Don't suppress the console output by running in standalone GUI mode when running a debug build so that we receive any panic backtraces. Signed-off-by: Jordan Yates --- src/main_gui.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_gui.rs b/src/main_gui.rs index b842f8f..d12a230 100644 --- a/src/main_gui.rs +++ b/src/main_gui.rs @@ -1,4 +1,4 @@ -#![windows_subsystem = "windows"] +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] use std::env; use std::sync::{Arc, Mutex}; From 3017c967595d4c2e7ed7dddde7d054805ef5a25f Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sat, 27 Jun 2026 15:27:58 +1000 Subject: [PATCH 2/5] scripts: tdf_decoder_build: support extension TDFs Support custom builds of the tool with extension TDF files from Infuse-IoT downstream users. Signed-off-by: Jordan Yates --- doc/custom_tdfs.md | 14 ++++++++++++++ scripts/tdf_decoder_build.py | 9 ++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 doc/custom_tdfs.md diff --git a/doc/custom_tdfs.md b/doc/custom_tdfs.md new file mode 100644 index 0000000..6d1444f --- /dev/null +++ b/doc/custom_tdfs.md @@ -0,0 +1,14 @@ +# Custom TDF Decoding + +To create a build of the tool with support for custom TDFs, the decoders need to be rebuilt +with `tdf_decoder_build.py` being provided the custom definition file. For example: + +``` +./scripts/tdf_decoder_build.py --json ./scripts/tdf.json --out ./tdf/src/ --extensions ~/code/extensions/tdf.json +``` + +This will update the `decoders_csv.rs` and `decoders_parquet.rs` files, and the GUI and CLI +applications can be rebuilt with a simple `cargo build --release`. + +> ⚠️ For MacOS builds, the resulting binaries must be notorized through Apple before + they can be run on other machines without warnings. diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index 1cab897..62cc133 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -739,9 +739,16 @@ def write_rendered(path, template): if __name__ == "__main__": parser = argparse.ArgumentParser("Generate rust TDF decoders", allow_abbrev=False) parser.add_argument("--json", required=True, type=str, help="TDF json description") + parser.add_argument("--extensions", type=str, help="Extension TDF json description") parser.add_argument("--out", required=True, type=str, help="Output folder") args = parser.parse_args() with open(args.json) as f: - definitions = json.load(f, parse_float=decimal.Decimal) + definitions: dict = json.load(f, parse_float=decimal.Decimal) + if args.extensions: + with open(args.extensions) as f: + extensions: dict = json.load(f, parse_float=decimal.Decimal) + definitions["structs"].update(extensions["structs"]) + definitions["definitions"].update(extensions["definitions"]) + decoders_gen(definitions, args.out) From 4a93ebda573af70d3e034aa16650083601181164 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sat, 27 Jun 2026 15:36:50 +1000 Subject: [PATCH 3/5] scritps: tdf_decoder_build: format file Apply `ruff` formatting to the generation script. Signed-off-by: Jordan Yates --- scripts/tdf_decoder_build.py | 51 ++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index 62cc133..c5061d3 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -269,7 +269,9 @@ def csv_flatten_field(field, convs, fmt, name_prefix=None, variable_item=False): for struct_field in structs[field["type"]]: convs += field_conv_func( struct_field, - name_prefix=field["name"] if name_prefix is None else f"{name_prefix}.{field['name']}", + name_prefix=field["name"] + if name_prefix is None + else f"{name_prefix}.{field['name']}", variable_item=variable_item, ) fmt += struct_fmts[field["type"]] @@ -321,9 +323,9 @@ def csv_field_byte_size(field, repeated_item=False): variable_field = None if info["fields"]: last_field = info["fields"][-1] - if ( - last_field.get("num", None) == 0 - and last_field["type"] not in ("char", "uint8_t") + if last_field.get("num", None) == 0 and last_field["type"] not in ( + "char", + "uint8_t", ): variable_field = last_field @@ -447,7 +449,9 @@ def field_byte_size(field): return field["num"] if c_type.startswith("struct "): struct_name = c_type.removeprefix("struct ") - return sum(field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"]) + return sum( + field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"] + ) if c_type == "char": return field.get("num", 0) base = { @@ -473,7 +477,10 @@ def field_model(field, path): if c_type.startswith("struct "): struct_name = c_type.removeprefix("struct ") child_fields = tdf_defs["structs"][struct_name]["fields"] - children = [field_model(child, path + [arrow_name(child["name"])]) for child in child_fields] + children = [ + field_model(child, path + [arrow_name(child["name"])]) + for child in child_fields + ] if num == 0: return { "kind": "list", @@ -508,8 +515,12 @@ def field_model(field, path): return { "kind": "list", "path": path, - "child": field_model({k: v for k, v in field.items() if k != "num"}, path), - "item_size": field_byte_size({k: v for k, v in field.items() if k != "num"}), + "child": field_model( + {k: v for k, v in field.items() if k != "num"}, path + ), + "item_size": field_byte_size( + {k: v for k, v in field.items() if k != "num"} + ), "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), } @@ -517,7 +528,9 @@ def field_model(field, path): return { "kind": "fixed_list", "path": path, - "child": field_model({k: v for k, v in field.items() if k != "num"}, path), + "child": field_model( + {k: v for k, v in field.items() if k != "num"}, path + ), "num": num, "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), } @@ -554,11 +567,17 @@ def model_storage_fields(model, out): def model_init_fields(model, out, capacity): kind = model["kind"] if kind == "primitive": - out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + out.append( + f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})" + ) elif kind == "string": - out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + out.append( + f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})" + ) elif kind == "binary": - out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + out.append( + f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})" + ) elif kind == "fixed_list": model_init_fields(model["child"], out, f"{capacity} * {model['num']}") elif kind == "list": @@ -647,8 +666,12 @@ def model_finish_expr(model): " }" ) if kind == "struct": - child_fields = ",\n ".join(child["field_expr"] for child in model["children"]) - child_arrays = ",\n ".join(model_finish_expr(child) for child in model["children"]) + child_fields = ",\n ".join( + child["field_expr"] for child in model["children"] + ) + child_arrays = ",\n ".join( + model_finish_expr(child) for child in model["children"] + ) return ( "Arc::new(StructArray::try_new(\n" f" Fields::from(vec![\n {child_fields}\n ]),\n" From 7d0724a5680d64ed6bc40a0e4b8603b7b4560f5e Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sat, 27 Jun 2026 16:01:55 +1000 Subject: [PATCH 4/5] scripts: tdf_decoder_build: support CSV hex conversion For TDF array fields with `"conversion": {"hex": true}`, output the decoded array as a single hex encoded string, instead of multiple independent columns. Signed-off-by: Jordan Yates --- scripts/tdf_decoder_build.py | 15 +++++++++++++++ scripts/tdf_decoder_csv.rs.jinja | 20 ++++++++++++++++++++ tdf/src/decoders_csv.rs | 19 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index c5061d3..d285ec6 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -178,6 +178,15 @@ def decoders_gen(tdf_defs, output): for _tdf_id, info in tdf_defs["definitions"].items(): info["arrow_schema"] = arrow_schema_expr(info, tdf_defs["structs"]) + def csv_fixed_hex_bytes(field): + conversion = field.get("conversion", {}) + return ( + field["type"] == "uint8_t" + and field.get("num", 0) > 0 + and conversion.get("hex", False) is True + and "int" not in conversion + ) + def field_conv_func(field, name_prefix=None, variable_item=False): t = rust_type[field["type"]] func = f"cursor.read_{t[0]}" @@ -227,6 +236,10 @@ def field_conv_func(field, name_prefix=None, variable_item=False): f"tdf_field_read_string_to_str(cursor, cursor_start, {field['num']}, size)?", ) ] + elif csv_fixed_hex_bytes(field): + return [ + (n, f"tdf_field_read_fixed_bytes_to_hex(cursor, {field['num']})?") + ] else: if field["num"] == 0: if variable_item: @@ -242,6 +255,8 @@ def field_conv_func(field, name_prefix=None, variable_item=False): def field_fmt(field): if field["type"] == "char": return ["{}"] + if csv_fixed_hex_bytes(field): + return ["{}"] if "display" in field and field["display"].get("fmt", "") == "hex": if digits := field["display"].get("digits", None): single = [f"0x{{:0{digits}x}}"] diff --git a/scripts/tdf_decoder_csv.rs.jinja b/scripts/tdf_decoder_csv.rs.jinja index e6bfe55..00566c1 100644 --- a/scripts/tdf_decoder_csv.rs.jinja +++ b/scripts/tdf_decoder_csv.rs.jinja @@ -29,6 +29,15 @@ fn tdf_field_read_vla_to_str(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size Ok(format!("{}", hex::encode(buf))) } +#[allow(dead_code)] +fn tdf_field_read_fixed_bytes_to_hex(cursor: &mut Cursor<&[u8]>, num: usize) -> Result +{ + let mut buf = vec![0u8; num]; + cursor.read_exact(&mut buf)?; + + Ok(format!("{}", hex::encode(buf))) +} + fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result { if (size as usize) < base_size { @@ -206,4 +215,15 @@ mod tests { assert_eq!(row, "0x12345678,9,abcdef"); } + + #[test] + fn conversion_hex_byte_array_uses_single_field_formatting() { + let bytes = [0xab, 0xcd, 0xef]; + let mut cursor = Cursor::new(bytes.as_slice()); + + let val = tdf_field_read_fixed_bytes_to_hex(&mut cursor, bytes.len()).unwrap(); + + assert_eq!(val, "abcdef"); + assert_eq!(cursor.position(), bytes.len() as u64); + } } diff --git a/tdf/src/decoders_csv.rs b/tdf/src/decoders_csv.rs index 081db72..8282e02 100644 --- a/tdf/src/decoders_csv.rs +++ b/tdf/src/decoders_csv.rs @@ -227,6 +227,14 @@ fn tdf_field_read_vla_to_str( Ok(format!("{}", hex::encode(buf))) } +#[allow(dead_code)] +fn tdf_field_read_fixed_bytes_to_hex(cursor: &mut Cursor<&[u8]>, num: usize) -> Result { + let mut buf = vec![0u8; num]; + cursor.read_exact(&mut buf)?; + + Ok(format!("{}", hex::encode(buf))) +} + fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result { if (size as usize) < base_size { return Result::Err(Error::new( @@ -887,4 +895,15 @@ mod tests { assert_eq!(row, "0x12345678,9,abcdef"); } + + #[test] + fn conversion_hex_byte_array_uses_single_field_formatting() { + let bytes = [0xab, 0xcd, 0xef]; + let mut cursor = Cursor::new(bytes.as_slice()); + + let val = tdf_field_read_fixed_bytes_to_hex(&mut cursor, bytes.len()).unwrap(); + + assert_eq!(val, "abcdef"); + assert_eq!(cursor.position(), bytes.len() as u64); + } } From 4d7f4bd3b3576976cb9dc094a00b82485ff69b2b Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Sat, 27 Jun 2026 16:04:04 +1000 Subject: [PATCH 5/5] main_cli: clarify `--path` argument Clarify the documentation of `--path` to show that it can be a path to a single file. Signed-off-by: Jordan Yates --- src/main_cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_cli.rs b/src/main_cli.rs index 3a7ec30..0105a75 100644 --- a/src/main_cli.rs +++ b/src/main_cli.rs @@ -45,7 +45,7 @@ impl infuse_decoder::ProgressReporter for IndicatifProgress { #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Cli { - /// The path to the folder containing Infuse-IoT binary files + /// The path to the file/folder containing Infuse-IoT binary files #[arg(short, long, required = true)] path: std::path::PathBuf, /// Output path for decoded files