Cargo.toml: v1.10.0

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 This project adheres to [Semantic Versioning](https://semver.org).
 
+## [1.10.0] - 2026-06-25
+
+ - Nested TDF definitions inherit the parent field name in the CSV header column
+ - Variable length array structs as the last element in a TDF generate a new row per instance
+ - Handle 0 length trailing VLAs
+
 ## [1.9.0] - 2026-06-16
 
  - New output format [Apache Parquet](https://parquet.apache.org/)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "infuse_decoder"
-version = "1.9.0"
+version = "1.10.0"
 edition = "2024"
 
 [[bin]]

scripts/tdf_decoder.rs.jinja

@@ -14,7 +14,7 @@ pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size:
 {
     let cursor_current = cursor.position();
     let cursor_read = cursor_current - cursor_start;
-    if cursor_read >= size as u64 {
+    if cursor_read > size as u64 {
         return Result::Err(Error::new(
             ErrorKind::InvalidData,
             "Insufficient data remaining",

scripts/tdf_decoder_build.py

@@ -178,7 +178,7 @@ def decoders_gen(tdf_defs, output):
     for _tdf_id, info in tdf_defs["definitions"].items():
         info["arrow_schema"] = arrow_schema_expr(info, tdf_defs["structs"])
 
-    def field_conv_func(field, name_prefix=None):
+    def field_conv_func(field, name_prefix=None, variable_item=False):
         t = rust_type[field["type"]]
         func = f"cursor.read_{t[0]}"
         if t[1]:
@@ -229,6 +229,8 @@ def field_conv_func(field, name_prefix=None):
                 ]
             else:
                 if field["num"] == 0:
+                    if variable_item:
+                        return [(n, func)]
                     return [
                         (n, "tdf_field_read_vla_to_str(cursor, cursor_start, size)?")
                     ]
@@ -256,27 +258,101 @@ def field_fmt(field):
     structs = {}
     struct_fmts = {}
     for name, struct in tdf_defs["structs"].items():
-        funcs = []
         fmts = []
         for f in struct["fields"]:
-            funcs += field_conv_func(f)
             fmts += field_fmt(f)
-        structs[f"struct {name}"] = funcs
+        structs[f"struct {name}"] = struct["fields"]
         struct_fmts[f"struct {name}"] = fmts
 
+    def csv_flatten_field(field, convs, fmt, name_prefix=None, variable_item=False):
+        if field["type"] in structs:
+            for struct_field in structs[field["type"]]:
+                convs += field_conv_func(
+                    struct_field,
+                    name_prefix=field["name"] if name_prefix is None else f"{name_prefix}.{field['name']}",
+                    variable_item=variable_item,
+                )
+            fmt += struct_fmts[field["type"]]
+        elif field["type"] in rust_type:
+            convs += field_conv_func(field, name_prefix, variable_item)
+            fmt_field = (
+                {k: v for k, v in field.items() if k != "num"}
+                if variable_item and field.get("num", None) == 0
+                else field
+            )
+            fmt += field_fmt(fmt_field)
+        else:
+            raise RuntimeError(f"Bad type '{field['type']}'")
+
+    def csv_field_byte_size(field, repeated_item=False):
+        c_type = field["type"]
+        conv = field.get("conversion", {})
+        if "int" in conv:
+            return field["num"]
+        if c_type in structs:
+            base = sum(csv_field_byte_size(child) for child in structs[c_type])
+            if not repeated_item and field.get("num", 1) not in (0, 1):
+                return base * field["num"]
+            return base
+        if c_type == "char":
+            return field.get("num", 0)
+        base = {
+            "int8_t": 1,
+            "uint8_t": 1,
+            "int16_t": 2,
+            "uint16_t": 2,
+            "int32_t": 4,
+            "uint32_t": 4,
+            "int64_t": 8,
+            "uint64_t": 8,
+            "float": 4,
+            "float32_t": 4,
+            "float64_t": 8,
+        }[c_type]
+        if not repeated_item and field.get("num", 1) not in (0, 1):
+            return base * field["num"]
+        return base
+
     # Generate rust conversion functions
     for _tdf_id, info in tdf_defs["definitions"].items():
         info["rust_convs"] = []
         fmt = []
-        for f in info["fields"]:
-            if f["type"] in structs:
-                info["rust_convs"] += structs[f["type"]]
-                fmt += struct_fmts[f["type"]]
-            elif f["type"] in rust_type:
-                info["rust_convs"] += field_conv_func(f)
-                fmt += field_fmt(f)
-            else:
-                raise RuntimeError(f"Bad type '{f['type']}'")
+        info["csv_variable"] = None
+        variable_field = None
+        if info["fields"]:
+            last_field = info["fields"][-1]
+            if (
+                last_field.get("num", None) == 0
+                and last_field["type"] not in ("char", "uint8_t")
+            ):
+                variable_field = last_field
+
+        fields = info["fields"][:-1] if variable_field is not None else info["fields"]
+        for f in fields:
+            csv_flatten_field(f, info["rust_convs"], fmt)
+
+        if variable_field is not None:
+            variable_convs = []
+            variable_fmt = []
+            csv_flatten_field(
+                variable_field,
+                variable_convs,
+                variable_fmt,
+                variable_item=True,
+            )
+            prefix_columns = len(info["rust_convs"]) + 1
+            info["csv_variable"] = {
+                "base_size": sum(csv_field_byte_size(field) for field in fields),
+                "item_size": csv_field_byte_size(variable_field, repeated_item=True),
+                "fmt": ",".join(variable_fmt),
+                "empty_fmt": ",".join(fmt + ["{}"] * len(variable_fmt)),
+                "base_convs": list(info["rust_convs"]),
+                "convs": variable_convs,
+                "empty_suffix": "," * len(variable_fmt),
+                "continuation_prefix": "," * prefix_columns,
+            }
+            info["rust_convs"] += variable_convs
+            fmt += variable_fmt
 
         info["rust_head"] = ",".join([f'"{c[0]}"' for c in info["rust_convs"]])
         info["rust_fmt"] = ",".join(fmt)

scripts/tdf_decoder_csv.rs.jinja

@@ -29,19 +29,72 @@ fn tdf_field_read_vla_to_str(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size
     Ok(format!("{}", hex::encode(buf)))
 }
 
+fn tdf_variable_item_count(size: u8, base_size: usize, item_size: usize) -> Result<usize>
+{
+    if (size as usize) < base_size {
+        return Result::Err(Error::new(
+            ErrorKind::InvalidData,
+            "Read underflow, corrupt data/metadata",
+        ));
+    }
+
+    let bytes_remaining = size as usize - base_size;
+    if bytes_remaining % item_size != 0 {
+        return Result::Err(Error::new(
+            ErrorKind::InvalidData,
+            "Variable-length array does not align to element size",
+        ));
+    }
+
+    Ok(bytes_remaining / item_size)
+}
+
 pub fn tdf_read_into_str(tdf_id: &u16, size: u8, cursor: &mut Cursor<&[u8]>) -> Result<String>
 {
     let cursor_start = cursor.position();
 
     let res = match tdf_id {
 {% for tdf_id, info in definitions.items() %}
+{% if info['csv_variable'] %}
+        {{ tdf_id }} => {
+            let item_count = tdf_variable_item_count(size, {{ info['csv_variable']['base_size'] }}, {{ info['csv_variable']['item_size'] }})?;
+            if item_count == 0 {
+                Ok(format!(
+                    "{{ info['csv_variable']['empty_fmt'] }}",
+{% for conv in info['csv_variable']['base_convs'] %}
+                    {{ conv[1] }},
+{% endfor %}
+{% for _conv in info['csv_variable']['convs'] %}
+                    "",
+{% endfor %}
+                ))
+            } else {
+                let mut out = format!(
+                    "{{ info['rust_fmt'] }}",
+{% for conv in info['rust_convs'] %}
+                    {{ conv[1] }},
+{% endfor %}
+                );
+                for _ in 1..item_count {
+                    out.push_str(&format!(
+                        "\n{{ info['csv_variable']['continuation_prefix'] }}{{ info['csv_variable']['fmt'] }}",
+{% for conv in info['csv_variable']['convs'] %}
+                        {{ conv[1] }},
+{% endfor %}
+                    ));
+                }
+                Ok(out)
+            }
+        },
+{% else %}
         {{ tdf_id }} =>
             Ok(format!(
                 "{{ info['rust_fmt'] }}",
 {% for conv in info['rust_convs'] %}
                 {{ conv[1] }},
 {% endfor %}
             )),
+{% endif %}
 {% endfor %}
         _ => {
             let mut buf = vec![0; size as usize];
@@ -68,3 +121,89 @@ pub fn tdf_read_into_str(tdf_id: &u16, size: u8, cursor: &mut Cursor<&[u8]>) ->
     }
     res
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tdf34_base_bytes() -> Vec<u8> {
+        let mut bytes = Vec::new();
+        bytes.extend_from_slice(&1u16.to_le_bytes());
+        bytes.extend_from_slice(&2u16.to_le_bytes());
+        bytes.extend_from_slice(&3u32.to_le_bytes());
+        bytes.extend_from_slice(&4u16.to_le_bytes());
+        bytes.extend_from_slice(&5u32.to_le_bytes());
+        bytes.push(6);
+        bytes.push((-7i8) as u8);
+        bytes
+    }
+
+    fn push_tdf34_neighbour(bytes: &mut Vec<u8>, earfcn: u32, pci: u16, time_diff: u16, rsrp: u8, rsrq: i8) {
+        bytes.extend_from_slice(&earfcn.to_le_bytes());
+        bytes.extend_from_slice(&pci.to_le_bytes());
+        bytes.extend_from_slice(&time_diff.to_le_bytes());
+        bytes.push(rsrp);
+        bytes.push(rsrq as u8);
+    }
+
+    #[test]
+    fn trailing_variable_array_zero_items_blanks_last_columns() {
+        let bytes = tdf34_base_bytes();
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let row = tdf_read_into_str(&34, bytes.len() as u8, &mut cursor).unwrap();
+
+        assert_eq!(row, "1,2,3,4,5,-6,-7,,,,,");
+    }
+
+    #[test]
+    fn trailing_variable_array_one_item_stays_on_first_row() {
+        let mut bytes = tdf34_base_bytes();
+        push_tdf34_neighbour(&mut bytes, 100, 11, 2500, 8, -9);
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let row = tdf_read_into_str(&34, bytes.len() as u8, &mut cursor).unwrap();
+
+        assert_eq!(row, "1,2,3,4,5,-6,-7,100,11,2.5,-8,-9");
+    }
+
+    #[test]
+    fn trailing_variable_array_extra_items_add_blank_prefix_rows() {
+        let mut bytes = tdf34_base_bytes();
+        push_tdf34_neighbour(&mut bytes, 100, 11, 2500, 8, -9);
+        push_tdf34_neighbour(&mut bytes, 200, 12, 3000, 10, -11);
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let row = tdf_read_into_str(&34, bytes.len() as u8, &mut cursor).unwrap();
+
+        assert_eq!(
+            row,
+            "1,2,3,4,5,-6,-7,100,11,2.5,-8,-9\n,,,,,,,,200,12,3,-10,-11"
+        );
+    }
+
+    #[test]
+    fn trailing_variable_array_items_use_single_field_formatting() {
+        let mut bytes = Vec::new();
+        bytes.extend_from_slice(&0x12345678u32.to_le_bytes());
+        bytes.extend_from_slice(&0x90abcdefu32.to_le_bytes());
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let row = tdf_read_into_str(&52, bytes.len() as u8, &mut cursor).unwrap();
+
+        assert_eq!(row, "0x12345678\n,0x90abcdef");
+    }
+
+    #[test]
+    fn trailing_uint8_variable_array_stays_as_hex_payload() {
+        let mut bytes = Vec::new();
+        bytes.extend_from_slice(&0x12345678u32.to_le_bytes());
+        bytes.extend_from_slice(&9u16.to_le_bytes());
+        bytes.extend_from_slice(&[0xab, 0xcd, 0xef]);
+        let mut cursor = Cursor::new(bytes.as_slice());
+
+        let row = tdf_read_into_str(&25, bytes.len() as u8, &mut cursor).unwrap();
+
+        assert_eq!(row, "0x12345678,9,abcdef");
+    }
+}

tdf/src/decoders.rs

@@ -1,7 +1,6 @@
-use std::io::{Cursor, Read, Result, Error, ErrorKind};
+use std::io::{Cursor, Error, ErrorKind, Read, Result};
 
-pub fn tdf_name(tdf_id: &u16) -> String
-{
+pub fn tdf_name(tdf_id: &u16) -> String {
     match tdf_id {
         1 => String::from("ANNOUNCE"),
         2 => String::from("BATTERY_STATE"),
@@ -67,11 +66,14 @@ pub fn tdf_name(tdf_id: &u16) -> String
     }
 }
 
-pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result<usize>
-{
+pub fn vla_bytes_remaining(
+    cursor: &mut Cursor<&[u8]>,
+    cursor_start: u64,
+    size: u8,
+) -> Result<usize> {
     let cursor_current = cursor.position();
     let cursor_read = cursor_current - cursor_start;
-    if cursor_read >= size as u64 {
+    if cursor_read > size as u64 {
         return Result::Err(Error::new(
             ErrorKind::InvalidData,
             "Insufficient data remaining",
@@ -82,8 +84,12 @@ pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size:
     Ok(bytes_remaining as usize)
 }
 
-pub fn tdf_field_read_string(cursor: &mut Cursor<&[u8]>, cursor_start: u64, num: u8, size: u8) ->  Result<Vec<u8>>
-{
+pub fn tdf_field_read_string(
+    cursor: &mut Cursor<&[u8]>,
+    cursor_start: u64,
+    num: u8,
+    size: u8,
+) -> Result<Vec<u8>> {
     let string_length = match num {
         0 => vla_bytes_remaining(cursor, cursor_start, size)?,
         _ => num as usize,
@@ -95,8 +101,11 @@ pub fn tdf_field_read_string(cursor: &mut Cursor<&[u8]>, cursor_start: u64, num:
     Ok(buf)
 }
 
-pub fn tdf_field_read_vla(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) ->  Result<Vec<u8>>
-{
+pub fn tdf_field_read_vla(
+    cursor: &mut Cursor<&[u8]>,
+    cursor_start: u64,
+    size: u8,
+) -> Result<Vec<u8>> {
     let bytes_remaining = vla_bytes_remaining(cursor, cursor_start, size)?;
     let mut buf = vec![0u8; bytes_remaining];