diff --git a/doc/custom_tdfs.md b/doc/custom_tdfs.md new file mode 100644 index 0000000..6d1444f --- /dev/null +++ b/doc/custom_tdfs.md @@ -0,0 +1,14 @@ +# Custom TDF Decoding + +To create a build of the tool with support for custom TDFs, the decoders need to be rebuilt +with `tdf_decoder_build.py` being provided the custom definition file. For example: + +``` +./scripts/tdf_decoder_build.py --json ./scripts/tdf.json --out ./tdf/src/ --extensions ~/code/extensions/tdf.json +``` + +This will update the `decoders_csv.rs` and `decoders_parquet.rs` files, and the GUI and CLI +applications can be rebuilt with a simple `cargo build --release`. + +> ⚠️ For MacOS builds, the resulting binaries must be notorized through Apple before + they can be run on other machines without warnings. diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index 1cab897..d285ec6 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -178,6 +178,15 @@ def decoders_gen(tdf_defs, output): for _tdf_id, info in tdf_defs["definitions"].items(): info["arrow_schema"] = arrow_schema_expr(info, tdf_defs["structs"]) + def csv_fixed_hex_bytes(field): + conversion = field.get("conversion", {}) + return ( + field["type"] == "uint8_t" + and field.get("num", 0) > 0 + and conversion.get("hex", False) is True + and "int" not in conversion + ) + def field_conv_func(field, name_prefix=None, variable_item=False): t = rust_type[field["type"]] func = f"cursor.read_{t[0]}" @@ -227,6 +236,10 @@ def field_conv_func(field, name_prefix=None, variable_item=False): f"tdf_field_read_string_to_str(cursor, cursor_start, {field['num']}, size)?", ) ] + elif csv_fixed_hex_bytes(field): + return [ + (n, f"tdf_field_read_fixed_bytes_to_hex(cursor, {field['num']})?") + ] else: if field["num"] == 0: if variable_item: @@ -242,6 +255,8 @@ def field_conv_func(field, name_prefix=None, variable_item=False): def field_fmt(field): if field["type"] == "char": return ["{}"] + if csv_fixed_hex_bytes(field): + return ["{}"] if "display" in field and field["display"].get("fmt", "") == "hex": if digits := field["display"].get("digits", None): single = [f"0x{{:0{digits}x}}"] @@ -269,7 +284,9 @@ def csv_flatten_field(field, convs, fmt, name_prefix=None, variable_item=False): for struct_field in structs[field["type"]]: convs += field_conv_func( struct_field, - name_prefix=field["name"] if name_prefix is None else f"{name_prefix}.{field['name']}", + name_prefix=field["name"] + if name_prefix is None + else f"{name_prefix}.{field['name']}", variable_item=variable_item, ) fmt += struct_fmts[field["type"]] @@ -321,9 +338,9 @@ def csv_field_byte_size(field, repeated_item=False): variable_field = None if info["fields"]: last_field = info["fields"][-1] - if ( - last_field.get("num", None) == 0 - and last_field["type"] not in ("char", "uint8_t") + if last_field.get("num", None) == 0 and last_field["type"] not in ( + "char", + "uint8_t", ): variable_field = last_field @@ -447,7 +464,9 @@ def field_byte_size(field): return field["num"] if c_type.startswith("struct "): struct_name = c_type.removeprefix("struct ") - return sum(field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"]) + return sum( + field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"] + ) if c_type == "char": return field.get("num", 0) base = { @@ -473,7 +492,10 @@ def field_model(field, path): if c_type.startswith("struct "): struct_name = c_type.removeprefix("struct ") child_fields = tdf_defs["structs"][struct_name]["fields"] - children = [field_model(child, path + [arrow_name(child["name"])]) for child in child_fields] + children = [ + field_model(child, path + [arrow_name(child["name"])]) + for child in child_fields + ] if num == 0: return { "kind": "list", @@ -508,8 +530,12 @@ def field_model(field, path): return { "kind": "list", "path": path, - "child": field_model({k: v for k, v in field.items() if k != "num"}, path), - "item_size": field_byte_size({k: v for k, v in field.items() if k != "num"}), + "child": field_model( + {k: v for k, v in field.items() if k != "num"}, path + ), + "item_size": field_byte_size( + {k: v for k, v in field.items() if k != "num"} + ), "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), } @@ -517,7 +543,9 @@ def field_model(field, path): return { "kind": "fixed_list", "path": path, - "child": field_model({k: v for k, v in field.items() if k != "num"}, path), + "child": field_model( + {k: v for k, v in field.items() if k != "num"}, path + ), "num": num, "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), } @@ -554,11 +582,17 @@ def model_storage_fields(model, out): def model_init_fields(model, out, capacity): kind = model["kind"] if kind == "primitive": - out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + out.append( + f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})" + ) elif kind == "string": - out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + out.append( + f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})" + ) elif kind == "binary": - out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + out.append( + f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})" + ) elif kind == "fixed_list": model_init_fields(model["child"], out, f"{capacity} * {model['num']}") elif kind == "list": @@ -647,8 +681,12 @@ def model_finish_expr(model): " }" ) if kind == "struct": - child_fields = ",\n ".join(child["field_expr"] for child in model["children"]) - child_arrays = ",\n ".join(model_finish_expr(child) for child in model["children"]) + child_fields = ",\n ".join( + child["field_expr"] for child in model["children"] + ) + child_arrays = ",\n ".join( + model_finish_expr(child) for child in model["children"] + ) return ( "Arc::new(StructArray::try_new(\n" f" Fields::from(vec![\n {child_fields}\n ]),\n" @@ -739,9 +777,16 @@ def write_rendered(path, template): if __name__ == "__main__": parser = argparse.ArgumentParser("Generate rust TDF decoders", allow_abbrev=False) parser.add_argument("--json", required=True, type=str, help="TDF json description") + parser.add_argument("--extensions", type=str, help="Extension TDF json description") parser.add_argument("--out", required=True, type=str, help="Output folder") args = parser.parse_args() with open(args.json) as f: - definitions = json.load(f, parse_float=decimal.Decimal) + definitions: dict = json.load(f, parse_float=decimal.Decimal) + if args.extensions: + with open(args.extensions) as f: + extensions: dict = json.load(f, parse_float=decimal.Decimal) + definitions["structs"].update(extensions["structs"]) + definitions["definitions"].update(extensions["definitions"]) + decoders_gen(definitions, args.out) diff --git a/scripts/tdf_decoder_csv.rs.jinja b/scripts/tdf_decoder_csv.rs.jinja index e6bfe55..00566c1 100644 --- a/scripts/tdf_decoder_csv.rs.jinja +++ b/scripts/tdf_decoder_csv.rs.jinja @@ -29,6 +29,15 @@ fn tdf_field_read_vla_to_str(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size Ok(format!("{}", hex::encode(buf))) } +#[allow(dead_code)] +fn tdf_field_read_fixed_bytes_to_hex(cursor: &mut Cursor<&[u8]>, num: usize) -> Result +{ + let mut buf = vec![0u8; num]; + cursor.read_exact(&mut buf)?; + + Ok(format!("{}", hex::encode(buf))) +} + fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result { if (size as usize) < base_size { @@ -206,4 +215,15 @@ mod tests { assert_eq!(row, "0x12345678,9,abcdef"); } + + #[test] + fn conversion_hex_byte_array_uses_single_field_formatting() { + let bytes = [0xab, 0xcd, 0xef]; + let mut cursor = Cursor::new(bytes.as_slice()); + + let val = tdf_field_read_fixed_bytes_to_hex(&mut cursor, bytes.len()).unwrap(); + + assert_eq!(val, "abcdef"); + assert_eq!(cursor.position(), bytes.len() as u64); + } } diff --git a/src/main_cli.rs b/src/main_cli.rs index 3a7ec30..0105a75 100644 --- a/src/main_cli.rs +++ b/src/main_cli.rs @@ -45,7 +45,7 @@ impl infuse_decoder::ProgressReporter for IndicatifProgress { #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Cli { - /// The path to the folder containing Infuse-IoT binary files + /// The path to the file/folder containing Infuse-IoT binary files #[arg(short, long, required = true)] path: std::path::PathBuf, /// Output path for decoded files diff --git a/src/main_gui.rs b/src/main_gui.rs index b842f8f..d12a230 100644 --- a/src/main_gui.rs +++ b/src/main_gui.rs @@ -1,4 +1,4 @@ -#![windows_subsystem = "windows"] +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] use std::env; use std::sync::{Arc, Mutex}; diff --git a/tdf/src/decoders_csv.rs b/tdf/src/decoders_csv.rs index 081db72..8282e02 100644 --- a/tdf/src/decoders_csv.rs +++ b/tdf/src/decoders_csv.rs @@ -227,6 +227,14 @@ fn tdf_field_read_vla_to_str( Ok(format!("{}", hex::encode(buf))) } +#[allow(dead_code)] +fn tdf_field_read_fixed_bytes_to_hex(cursor: &mut Cursor<&[u8]>, num: usize) -> Result { + let mut buf = vec![0u8; num]; + cursor.read_exact(&mut buf)?; + + Ok(format!("{}", hex::encode(buf))) +} + fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result { if (size as usize) < base_size { return Result::Err(Error::new( @@ -887,4 +895,15 @@ mod tests { assert_eq!(row, "0x12345678,9,abcdef"); } + + #[test] + fn conversion_hex_byte_array_uses_single_field_formatting() { + let bytes = [0xab, 0xcd, 0xef]; + let mut cursor = Cursor::new(bytes.as_slice()); + + let val = tdf_field_read_fixed_bytes_to_hex(&mut cursor, bytes.len()).unwrap(); + + assert_eq!(val, "abcdef"); + assert_eq!(cursor.position(), bytes.len() as u64); + } }