From 342ca2ba571e70afc8d48edfdc5a321ae9cb6d65 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Tue, 14 Jul 2026 10:36:26 +1000 Subject: [PATCH 1/8] scripts: tdf_decoder_build.py: smaller parquet types For TDF fields with an integer conversion type, use smaller integer types when possible instead of `Float64`. This reduces the size of the Parquet files. Signed-off-by: Jordan Yates --- CHANGELOG.md | 4 ++ scripts/tdf_decoder_build.py | 118 +++++++++++++++++++++++++++++++++-- tdf/src/decoders.rs | 27 +++----- tdf/src/decoders_parquet.rs | 29 +++++---- 4 files changed, 141 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1db764..a10758c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org). +## [1.12.0] - xx-xx-xx + + - Tighten Parquet output types for field conversion + ## [1.11.0] - 2026-06-30 - Output file list is now scrollable diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index d285ec6..4d89921 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -43,6 +43,39 @@ "float64_t": "DataType::Float64", } +integer_range = { + "int8_t": (-(2**7), 2**7 - 1), + "uint8_t": (0, 2**8 - 1), + "int16_t": (-(2**15), 2**15 - 1), + "uint16_t": (0, 2**16 - 1), + "int32_t": (-(2**31), 2**31 - 1), + "uint32_t": (0, 2**32 - 1), + "int64_t": (-(2**63), 2**63 - 1), + "uint64_t": (0, 2**64 - 1), +} + +rust_int_range = { + "i8": (-(2**7), 2**7 - 1), + "u8": (0, 2**8 - 1), + "i16": (-(2**15), 2**15 - 1), + "u16": (0, 2**16 - 1), + "i32": (-(2**31), 2**31 - 1), + "u32": (0, 2**32 - 1), + "i64": (-(2**63), 2**63 - 1), + "u64": (0, 2**64 - 1), +} + +arrow_rust_int_type = { + "i8": "DataType::Int8", + "u8": "DataType::UInt8", + "i16": "DataType::Int16", + "u16": "DataType::UInt16", + "i32": "DataType::Int32", + "u32": "DataType::UInt32", + "i64": "DataType::Int64", + "u64": "DataType::UInt64", +} + def rust_str(value): return json.dumps(value) @@ -60,6 +93,57 @@ def indent_block(value, indent): return "\n".join(f"{pad}{line}" for line in value.splitlines()) +def integer_type_for_range(min_val, max_val): + if min_val < 0: + candidates = ("i8", "i16", "i32", "i64") + else: + candidates = ("u8", "u16", "u32", "u64") + + for type_name in candidates: + type_min, type_max = rust_int_range[type_name] + if type_min <= min_val and max_val <= type_max: + return type_name + + return None + + +def decimal_value(value): + if isinstance(value, decimal.Decimal): + return value + return decimal.Decimal(value) + + +def integral_decimal(value): + value = decimal_value(value) + if value == value.to_integral_value(): + return int(value) + return None + + +def raw_integer_range(field): + conv = field.get("conversion", {}) + if "int" in conv: + byte_len = field["num"] + return (0, 2 ** (byte_len * 8) - 1) + return integer_range.get(field["type"]) + + +def integer_type_after_conversion(field): + value_range = raw_integer_range(field) + if value_range is None: + return None + + conv = field.get("conversion", {}) + m = integral_decimal(conv.get("m", 1)) + c = integral_decimal(conv.get("c", 0)) + if m is None or c is None: + return None + + min_val, max_val = value_range + converted = (min_val * m + c, max_val * m + c) + return integer_type_for_range(min(converted), max(converted)) + + def arrow_scalar_type(field): c_type = field["type"] conv = field.get("conversion", {}) @@ -68,6 +152,9 @@ def arrow_scalar_type(field): return "DataType::Utf8" if "m" in conv or "c" in conv: + int_type = integer_type_after_conversion(field) + if int_type is not None: + return arrow_rust_int_type[int_type] return "DataType::Float64" if "int" in conv: @@ -115,7 +202,7 @@ def arrow_data_type_expr(field, structs, indent): num = field["num"] if num == 0: - if c_type == "uint8_t": + if c_type == "uint8_t" and not conv: return "DataType::Binary" return ( "DataType::List(Arc::new(Field::new_list_field(\n" @@ -124,7 +211,7 @@ def arrow_data_type_expr(field, structs, indent): f"{' ' * indent})))" ) - if c_type == "uint8_t": + if c_type == "uint8_t" and not conv: return f"DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::UInt8, false)), {num})" return ( "DataType::FixedSizeList(Arc::new(Field::new_list_field(\n" @@ -405,6 +492,9 @@ def csv_field_byte_size(field, repeated_item=False): def rust_type_after_conversion(field): conv = field.get("conversion", {}) if "m" in conv or "c" in conv: + int_type = integer_type_after_conversion(field) + if int_type is not None: + return int_type return "f64" if field["type"] == "char": return "String" @@ -445,9 +535,29 @@ def primitive_read_expr(field): func = f"cursor.read_{t_name}::<{e}>()?" if "m" in c or "c" in c: + int_type = integer_type_after_conversion(field) + if int_type is not None: + scale = integral_decimal(c.get("m", 1)) + offset = integral_decimal(c.get("c", 0)) + + if scale == 0: + func = f"({offset} as {int_type})" + else: + func = f"({func} as {int_type})" + if scale != 1: + func += f" * {scale}" + if offset > 0: + func += f" + {offset}" + elif offset < 0: + func += f" - {-offset}" + return func + func += " as f64" - if "m" in c and c["m"] != 0: + if "m" in c and c["m"] != 1: val = c["m"] + if val == 0: + func += " * 0.0" + return func inverse_ratio = (1 / val).as_integer_ratio() if inverse_ratio[1] == 1: func += f" / {inverse_ratio[0]}.0" @@ -519,7 +629,7 @@ def field_model(field, path): "read": f"tdf_field_read_string_to_string(cursor, cursor_start, {num or 0}, size)?", } - if num == 0 and c_type == "uint8_t" and "int" not in conv: + if num == 0 and c_type == "uint8_t" and not conv: return { "kind": "binary", "path": path, diff --git a/tdf/src/decoders.rs b/tdf/src/decoders.rs index 32cb6b1..b46526d 100644 --- a/tdf/src/decoders.rs +++ b/tdf/src/decoders.rs @@ -1,6 +1,7 @@ -use std::io::{Cursor, Error, ErrorKind, Read, Result}; +use std::io::{Cursor, Read, Result, Error, ErrorKind}; -pub fn tdf_name(tdf_id: &u16) -> String { +pub fn tdf_name(tdf_id: &u16) -> String +{ match tdf_id { 1 => String::from("ANNOUNCE"), 2 => String::from("BATTERY_STATE"), @@ -66,11 +67,8 @@ pub fn tdf_name(tdf_id: &u16) -> String { } } -pub fn vla_bytes_remaining( - cursor: &mut Cursor<&[u8]>, - cursor_start: u64, - size: u8, -) -> Result { +pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result +{ let cursor_current = cursor.position(); let cursor_read = cursor_current - cursor_start; if cursor_read > size as u64 { @@ -84,12 +82,8 @@ pub fn vla_bytes_remaining( Ok(bytes_remaining as usize) } -pub fn tdf_field_read_string( - cursor: &mut Cursor<&[u8]>, - cursor_start: u64, - num: u8, - size: u8, -) -> Result> { +pub fn tdf_field_read_string(cursor: &mut Cursor<&[u8]>, cursor_start: u64, num: u8, size: u8) -> Result> +{ let string_length = match num { 0 => vla_bytes_remaining(cursor, cursor_start, size)?, _ => num as usize, @@ -101,11 +95,8 @@ pub fn tdf_field_read_string( Ok(buf) } -pub fn tdf_field_read_vla( - cursor: &mut Cursor<&[u8]>, - cursor_start: u64, - size: u8, -) -> Result> { +pub fn tdf_field_read_vla(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result> +{ let bytes_remaining = vla_bytes_remaining(cursor, cursor_start, size)?; let mut buf = vec![0u8; bytes_remaining]; diff --git a/tdf/src/decoders_parquet.rs b/tdf/src/decoders_parquet.rs index ecbf66f..6ed13a1 100644 --- a/tdf/src/decoders_parquet.rs +++ b/tdf/src/decoders_parquet.rs @@ -480,7 +480,7 @@ pub fn tdf_parquet_schema(tdf_id: u16) -> Option { Field::new("earfcn", DataType::UInt32, false), Field::new("status", DataType::UInt8, false), Field::new("tech", DataType::UInt8, false), - Field::new("rsrp", DataType::Float64, false), + Field::new("rsrp", DataType::Int16, false), Field::new("rsrq", DataType::Int8, false), ]))), 22 => Some(Arc::new(Schema::new(vec![ @@ -595,7 +595,7 @@ pub fn tdf_parquet_schema(tdf_id: u16) -> Option { false, ), Field::new("earfcn", DataType::UInt32, false), - Field::new("rsrp", DataType::Float64, false), + Field::new("rsrp", DataType::Int16, false), Field::new("rsrq", DataType::Int8, false), Field::new( "neighbours", @@ -604,7 +604,7 @@ pub fn tdf_parquet_schema(tdf_id: u16) -> Option { Field::new("earfcn", DataType::UInt32, false), Field::new("pci", DataType::UInt16, false), Field::new("time_diff", DataType::Float64, false), - Field::new("rsrp", DataType::Float64, false), + Field::new("rsrp", DataType::Int16, false), Field::new("rsrq", DataType::Int8, false), ])), false, @@ -2870,7 +2870,7 @@ pub struct Tdf21LteConnStatusBuilder { earfcn: Vec, status: Vec, tech: Vec, - rsrp: Vec, + rsrp: Vec, rsrq: Vec, } @@ -2916,7 +2916,7 @@ impl Tdf21LteConnStatusBuilder { self.earfcn.push(cursor.read_u32::()?); self.status.push(cursor.read_u8()?); self.tech.push(cursor.read_u8()?); - self.rsrp.push(cursor.read_u8()? as f64 / -1.0); + self.rsrp.push((cursor.read_u8()? as i16) * -1); self.rsrq.push(cursor.read_i8()?); finish_tdf_read(cursor, cursor_start, size) @@ -2948,7 +2948,7 @@ impl Tdf21LteConnStatusBuilder { Arc::new(UInt32Array::from(std::mem::take(&mut self.earfcn))) as ArrayRef, Arc::new(UInt8Array::from(std::mem::take(&mut self.status))) as ArrayRef, Arc::new(UInt8Array::from(std::mem::take(&mut self.tech))) as ArrayRef, - Arc::new(Float64Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, Arc::new(Int8Array::from(std::mem::take(&mut self.rsrq))) as ArrayRef, ]; @@ -3731,13 +3731,13 @@ pub struct Tdf34LteTacCellsBuilder { cell_eci: Vec, cell_tac: Vec, earfcn: Vec, - rsrp: Vec, + rsrp: Vec, rsrq: Vec, neighbours_offsets: Vec, neighbours_earfcn: Vec, neighbours_pci: Vec, neighbours_time_diff: Vec, - neighbours_rsrp: Vec, + neighbours_rsrp: Vec, neighbours_rsrq: Vec, } @@ -3793,7 +3793,7 @@ impl Tdf34LteTacCellsBuilder { self.cell_eci.push(cursor.read_u32::()?); self.cell_tac.push(cursor.read_u16::()?); self.earfcn.push(cursor.read_u32::()?); - self.rsrp.push(cursor.read_u8()? as f64 / -1.0); + self.rsrp.push((cursor.read_u8()? as i16) * -1); self.rsrq.push(cursor.read_i8()?); { let bytes_remaining = crate::decoders::vla_bytes_remaining(cursor, cursor_start, size)?; @@ -3810,7 +3810,7 @@ impl Tdf34LteTacCellsBuilder { self.neighbours_pci.push(cursor.read_u16::()?); self.neighbours_time_diff .push(cursor.read_u16::()? as f64 / 1000.0); - self.neighbours_rsrp.push(cursor.read_u8()? as f64 / -1.0); + self.neighbours_rsrp.push((cursor.read_u8()? as i16) * -1); self.neighbours_rsrq.push(cursor.read_i8()?); } self.neighbours_offsets @@ -3844,7 +3844,7 @@ impl Tdf34LteTacCellsBuilder { None, )?) as ArrayRef, Arc::new(UInt32Array::from(std::mem::take(&mut self.earfcn))) as ArrayRef, - Arc::new(Float64Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, Arc::new(Int8Array::from(std::mem::take(&mut self.rsrq))) as ArrayRef, { let offsets = std::mem::replace(&mut self.neighbours_offsets, vec![0]); @@ -3856,7 +3856,7 @@ impl Tdf34LteTacCellsBuilder { Field::new("earfcn", DataType::UInt32, false), Field::new("pci", DataType::UInt16, false), Field::new("time_diff", DataType::Float64, false), - Field::new("rsrp", DataType::Float64, false), + Field::new("rsrp", DataType::Int16, false), Field::new("rsrq", DataType::Int8, false), ]), vec![ @@ -3868,9 +3868,8 @@ impl Tdf34LteTacCellsBuilder { Arc::new(Float64Array::from(std::mem::take( &mut self.neighbours_time_diff, ))) as ArrayRef, - Arc::new(Float64Array::from(std::mem::take( - &mut self.neighbours_rsrp, - ))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.neighbours_rsrp))) + as ArrayRef, Arc::new(Int8Array::from(std::mem::take(&mut self.neighbours_rsrq))) as ArrayRef, ], From 5d6129bba60e1137bb9efefd70ea002d09486ed2 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 10:53:03 +1000 Subject: [PATCH 2/8] tdf: lib: allow trailing minimally sized TDF Fix decoding of a TDF block that contains a 3 byte TDF at the end of the block. A 3 byte TDF can be constructed with an ID, no timestamp, and a single byte of data. Signed-off-by: Jordan Yates --- CHANGELOG.md | 1 + tdf/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a10758c..1dded4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](https://semver.org). ## [1.12.0] - xx-xx-xx - Tighten Parquet output types for field conversion + - Fix decoding minimally sized last TDF ## [1.11.0] - 2026-06-30 diff --git a/tdf/src/lib.rs b/tdf/src/lib.rs index f78f7bd..753461c 100644 --- a/tdf/src/lib.rs +++ b/tdf/src/lib.rs @@ -145,7 +145,7 @@ pub fn block_decode( let mut cursor = Cursor::new(block); let mut buffer_time: i64 = 0; - while block.len() - cursor.position() as usize > 4 { + while block.len() - cursor.position() as usize > 3 { let header = cursor.read_u16::()?; if header == 0xFFFF || header == 0x0000 { break; From 70c6f6958a2fe45053a2f53de0e36a8ae2deedfc Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 10:59:15 +1000 Subject: [PATCH 3/8] scripts: tdf_decoder_build.py: autoformat generated files Automatically format generated rust files with `rustfmt`. Signed-off-by: Jordan Yates --- scripts/tdf_decoder_build.py | 2 ++ tdf/src/decoders.rs | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index 4d89921..a30c028 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -6,6 +6,7 @@ import os import pathlib import re +import subprocess from numpy import format_float_positional as float_format from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -878,6 +879,7 @@ def write_rendered(path, template): ) f.write(rendered) f.write(os.linesep) + subprocess.run(["rustfmt", path], check=True) write_rendered(common_output, common_template) write_rendered(csv_output, csv_template) diff --git a/tdf/src/decoders.rs b/tdf/src/decoders.rs index b46526d..32cb6b1 100644 --- a/tdf/src/decoders.rs +++ b/tdf/src/decoders.rs @@ -1,7 +1,6 @@ -use std::io::{Cursor, Read, Result, Error, ErrorKind}; +use std::io::{Cursor, Error, ErrorKind, Read, Result}; -pub fn tdf_name(tdf_id: &u16) -> String -{ +pub fn tdf_name(tdf_id: &u16) -> String { match tdf_id { 1 => String::from("ANNOUNCE"), 2 => String::from("BATTERY_STATE"), @@ -67,8 +66,11 @@ pub fn tdf_name(tdf_id: &u16) -> String } } -pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result -{ +pub fn vla_bytes_remaining( + cursor: &mut Cursor<&[u8]>, + cursor_start: u64, + size: u8, +) -> Result { let cursor_current = cursor.position(); let cursor_read = cursor_current - cursor_start; if cursor_read > size as u64 { @@ -82,8 +84,12 @@ pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: Ok(bytes_remaining as usize) } -pub fn tdf_field_read_string(cursor: &mut Cursor<&[u8]>, cursor_start: u64, num: u8, size: u8) -> Result> -{ +pub fn tdf_field_read_string( + cursor: &mut Cursor<&[u8]>, + cursor_start: u64, + num: u8, + size: u8, +) -> Result> { let string_length = match num { 0 => vla_bytes_remaining(cursor, cursor_start, size)?, _ => num as usize, @@ -95,8 +101,11 @@ pub fn tdf_field_read_string(cursor: &mut Cursor<&[u8]>, cursor_start: u64, num: Ok(buf) } -pub fn tdf_field_read_vla(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result> -{ +pub fn tdf_field_read_vla( + cursor: &mut Cursor<&[u8]>, + cursor_start: u64, + size: u8, +) -> Result> { let bytes_remaining = vla_bytes_remaining(cursor, cursor_start, size)?; let mut buf = vec![0u8; bytes_remaining]; From b1444a49845f6a10616a208e7a723adcd724b6f8 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 10:59:47 +1000 Subject: [PATCH 4/8] scripts: tdf.json: update definitions Pull in latest definitions from Infuse-SDK and regenerate decoders. Signed-off-by: Jordan Yates --- scripts/tdf.json | 20 +++++++++++ tdf/src/decoders.rs | 1 + tdf/src/decoders_csv.rs | 6 ++++ tdf/src/decoders_parquet.rs | 69 +++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) diff --git a/scripts/tdf.json b/scripts/tdf.json index d01316b..55a083f 100644 --- a/scripts/tdf.json +++ b/scripts/tdf.json @@ -1959,6 +1959,26 @@ "description": "New data value, empty for delete, '*' for write-only" } ] + }, + "62": { + "name": "AMBIENT_PRESSURE", + "description": "Ambient pressure", + "fields": [ + { + "name": "pressure", + "type": "uint32_t", + "description": "Atmospheric pressure (pascals)", + "display": { + "fmt": "float", + "digits": 3, + "postfix": "kPA" + }, + "conversion": { + "m": 0.001, + "c": 0 + } + } + ] } } } \ No newline at end of file diff --git a/tdf/src/decoders.rs b/tdf/src/decoders.rs index 32cb6b1..1bf6bf8 100644 --- a/tdf/src/decoders.rs +++ b/tdf/src/decoders.rs @@ -62,6 +62,7 @@ pub fn tdf_name(tdf_id: &u16) -> String { 59 => String::from("PCM_16BIT_CHAN_RIGHT"), 60 => String::from("PCM_16BIT_CHAN_DUAL"), 61 => String::from("KVS_VALUE_CHANGED"), + 62 => String::from("AMBIENT_PRESSURE"), _ => format!("{}", tdf_id), } } diff --git a/tdf/src/decoders_csv.rs b/tdf/src/decoders_csv.rs index 8282e02..a8de577 100644 --- a/tdf/src/decoders_csv.rs +++ b/tdf/src/decoders_csv.rs @@ -199,6 +199,7 @@ pub fn tdf_fields(tdf_id: &u16) -> Vec<&'static str> { 59 => vec!["val"], 60 => vec!["left", "right"], 61 => vec!["key", "value"], + 62 => vec!["pressure"], _ => vec!["unknown"], } } @@ -778,6 +779,11 @@ pub fn tdf_read_into_str(tdf_id: &u16, size: u8, cursor: &mut Cursor<&[u8]>) -> cursor.read_u16::()?, tdf_field_read_vla_to_str(cursor, cursor_start, size)?, )), + 62 => + Ok(format!( + "{}", + cursor.read_u32::()? as f64 / 1000.0, + )), _ => { let mut buf = vec![0; size as usize]; cursor.read_exact(&mut buf)?; diff --git a/tdf/src/decoders_parquet.rs b/tdf/src/decoders_parquet.rs index 6ed13a1..b6e8055 100644 --- a/tdf/src/decoders_parquet.rs +++ b/tdf/src/decoders_parquet.rs @@ -138,6 +138,7 @@ pub fn tdf_parquet_schemas() -> Vec<(u16, &'static str, SchemaRef)> { (59, "PCM_16BIT_CHAN_RIGHT", tdf_parquet_schema(59).unwrap()), (60, "PCM_16BIT_CHAN_DUAL", tdf_parquet_schema(60).unwrap()), (61, "KVS_VALUE_CHANGED", tdf_parquet_schema(61).unwrap()), + (62, "AMBIENT_PRESSURE", tdf_parquet_schema(62).unwrap()), ] } @@ -203,6 +204,7 @@ pub fn tdf_parquet_has_schema(tdf_id: u16) -> bool { 59 => true, 60 => true, 61 => true, + 62 => true, _ => false, } } @@ -808,6 +810,11 @@ pub fn tdf_parquet_schema(tdf_id: u16) -> Option { Field::new("key", DataType::UInt16, false), Field::new("value", DataType::Binary, false), ]))), + 62 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("pressure", DataType::Float64, false), + ]))), _ => None, } } @@ -994,6 +1001,9 @@ pub fn tdf_parquet_builder(tdf_id: u16, capacity: usize) -> Option Some(TdfParquetBatchBuilder::Tdf61KvsValueChanged( Tdf61KvsValueChangedBuilder::new(capacity), )), + 62 => Some(TdfParquetBatchBuilder::Tdf62AmbientPressure( + Tdf62AmbientPressureBuilder::new(capacity), + )), _ => None, } } @@ -1065,6 +1075,7 @@ pub enum TdfParquetBatchBuilder { Tdf59Pcm16bitChanRight(Tdf59Pcm16bitChanRightBuilder), Tdf60Pcm16bitChanDual(Tdf60Pcm16bitChanDualBuilder), Tdf61KvsValueChanged(Tdf61KvsValueChangedBuilder), + Tdf62AmbientPressure(Tdf62AmbientPressureBuilder), } impl TdfParquetBatchBuilder { @@ -1130,6 +1141,7 @@ impl TdfParquetBatchBuilder { Self::Tdf59Pcm16bitChanRight(builder) => builder.schema(), Self::Tdf60Pcm16bitChanDual(builder) => builder.schema(), Self::Tdf61KvsValueChanged(builder) => builder.schema(), + Self::Tdf62AmbientPressure(builder) => builder.schema(), } } @@ -1195,6 +1207,7 @@ impl TdfParquetBatchBuilder { Self::Tdf59Pcm16bitChanRight(builder) => builder.rows(), Self::Tdf60Pcm16bitChanDual(builder) => builder.rows(), Self::Tdf61KvsValueChanged(builder) => builder.rows(), + Self::Tdf62AmbientPressure(builder) => builder.rows(), } } @@ -1265,6 +1278,7 @@ impl TdfParquetBatchBuilder { Self::Tdf59Pcm16bitChanRight(builder) => builder.append(meta, size, cursor), Self::Tdf60Pcm16bitChanDual(builder) => builder.append(meta, size, cursor), Self::Tdf61KvsValueChanged(builder) => builder.append(meta, size, cursor), + Self::Tdf62AmbientPressure(builder) => builder.append(meta, size, cursor), } } @@ -1330,6 +1344,7 @@ impl TdfParquetBatchBuilder { Self::Tdf59Pcm16bitChanRight(builder) => builder.finish_batch(), Self::Tdf60Pcm16bitChanDual(builder) => builder.finish_batch(), Self::Tdf61KvsValueChanged(builder) => builder.finish_batch(), + Self::Tdf62AmbientPressure(builder) => builder.finish_batch(), } } } @@ -5590,3 +5605,57 @@ impl Tdf61KvsValueChangedBuilder { RecordBatch::try_new(schema, columns) } } + +pub struct Tdf62AmbientPressureBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + pressure: Vec, +} + +impl Tdf62AmbientPressureBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + pressure: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(62).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.pressure + .push(cursor.read_u32::()? as f64 / 1000.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.pressure))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} From cc58348c94c3448d44995b0160eb12253fabf3eb Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 11:01:25 +1000 Subject: [PATCH 5/8] tdf: lib: reject zero length index arrays Reject index arrays with zero length, the same way time arrays are handled. Signed-off-by: Jordan Yates --- CHANGELOG.md | 1 + tdf/src/lib.rs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dded4a..67dfb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org). - Tighten Parquet output types for field conversion - Fix decoding minimally sized last TDF + - Zero length index arrays are now rejected by the decoder ## [1.11.0] - 2026-06-30 diff --git a/tdf/src/lib.rs b/tdf/src/lib.rs index 753461c..5df445c 100644 --- a/tdf/src/lib.rs +++ b/tdf/src/lib.rs @@ -240,6 +240,13 @@ pub fn block_decode( } TDF_ARRAY_IDX => { array_num = cursor.read_u8()?; + if array_num == 0 { + // Invalid header, remainder of block can't be trusted + return std::io::Result::Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Time array of 0 elements", + )); + } array_sample_idx = Some(cursor.read_u16::()?); } _ => { From 1676e3f1a6059b9cd380d42304ce824eca4b5f9a Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 11:11:35 +1000 Subject: [PATCH 6/8] tdf: lib: reject relative timestamps without an absolute reference Reject relative timestamps in the decoder if they are not preceded by an absolute timestamp. Signed-off-by: Jordan Yates --- CHANGELOG.md | 1 + tdf/src/lib.rs | 114 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67dfb1b..a3ab41a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org). - Tighten Parquet output types for field conversion - Fix decoding minimally sized last TDF - Zero length index arrays are now rejected by the decoder + - Relative timestamps without a preceding absolute timestamp are now rejected ## [1.11.0] - 2026-06-30 diff --git a/tdf/src/lib.rs b/tdf/src/lib.rs index 5df445c..c5c3b82 100644 --- a/tdf/src/lib.rs +++ b/tdf/src/lib.rs @@ -144,6 +144,7 @@ pub fn block_decode( ) -> std::io::Result<()> { let mut cursor = Cursor::new(block); let mut buffer_time: i64 = 0; + let mut has_absolute_time = false; while block.len() - cursor.position() as usize > 3 { let header = cursor.read_u16::()?; @@ -170,9 +171,26 @@ pub fn block_decode( TDF_TIME_GLOBAL => { buffer_time = ((cursor.read_u32::()? as i64) << 16) + (cursor.read_u16::()? as i64); + has_absolute_time = true; + } + TDF_TIME_RELATIVE_U16 => { + if !has_absolute_time { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "Relative timestamp encountered before an absolute timestamp", + )); + } + buffer_time += cursor.read_u16::()? as i64; + } + TDF_TIME_RELATIVE_S24 => { + if !has_absolute_time { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "Extended relative timestamp encountered before an absolute timestamp", + )); + } + buffer_time += cursor.read_i24::()? as i64; } - TDF_TIME_RELATIVE_U16 => buffer_time += cursor.read_u16::()? as i64, - TDF_TIME_RELATIVE_S24 => buffer_time += cursor.read_i24::()? as i64, _ => { panic!("How?"); } @@ -291,3 +309,95 @@ pub fn block_decode( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[derive(Default)] + struct TestOutput { + written: HashMap<(Option, u16), usize>, + } + + impl TdfOutput for TestOutput { + fn write( + &mut self, + remote_id: Option, + tdf_id: u16, + _tdf_time: i64, + _tdf_idx: Option, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> std::io::Result<()> { + let mut buf = vec![0; size as usize]; + cursor.read_exact(&mut buf)?; + *self.written.entry((remote_id, tdf_id)).or_default() += 1; + Ok(()) + } + + fn iter_written(&self) -> impl Iterator, u16), &usize)> { + self.written.iter() + } + + fn written(&self, remote_id: Option, tdf_id: u16) -> usize { + self.written.get(&(remote_id, tdf_id)).copied().unwrap_or(0) + } + + fn output_path(self: &Self, _remote_id: Option, _tdf_id: u16) -> Option { + None + } + } + + #[test] + fn rejects_relative_u16_timestamp_before_absolute_timestamp() { + let block = [ + 0xE7, 0x83, // id=999, relative u16 timestamp + 0x01, // size + 0x34, 0x12, // relative timestamp + 0xAA, // payload + ]; + let mut output = TestOutput::default(); + + let error = block_decode(None, &block, &mut output).unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::InvalidData); + assert_eq!(output.written(None, 999), 0); + } + + #[test] + fn rejects_relative_s24_timestamp_before_absolute_timestamp() { + let block = [ + 0xE7, 0xC3, // id=999, relative s24 timestamp + 0x01, // size + 0x01, 0x02, 0x03, // relative timestamp + 0xAA, // payload + ]; + let mut output = TestOutput::default(); + + let error = block_decode(None, &block, &mut output).unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::InvalidData); + assert_eq!(output.written(None, 999), 0); + } + + #[test] + fn accepts_relative_timestamp_after_absolute_timestamp() { + let block = [ + 0xE7, 0x43, // id=999, global timestamp + 0x01, // size + 0x78, 0x56, 0x34, 0x12, // timestamp seconds + 0xBC, 0x9A, // timestamp subsecond + 0xAA, // payload + 0xE7, 0x83, // id=999, relative u16 timestamp + 0x01, // size + 0x34, 0x12, // relative timestamp + 0xBB, // payload + ]; + let mut output = TestOutput::default(); + + block_decode(None, &block, &mut output).unwrap(); + + assert_eq!(output.written(None, 999), 2); + } +} From cb8b17698e89b757a65057f62105ed2d902e6ae7 Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 11:22:52 +1000 Subject: [PATCH 7/8] main_gui: fix compiler type warning The updated compiler triggers warnings on f32 vs f64 types in `egui::Stroke::new`. Signed-off-by: Jordan Yates --- src/main_gui.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_gui.rs b/src/main_gui.rs index 9a0aff5..aa1d1b1 100644 --- a/src/main_gui.rs +++ b/src/main_gui.rs @@ -275,7 +275,7 @@ fn draw_doc_marker(painter: &egui::Painter, marker: &DocMarker) { painter.circle_stroke( position, radius, - egui::Stroke::new(1.5, egui::Color32::WHITE), + egui::Stroke::new(1.5_f32, egui::Color32::WHITE), ); painter.text( position, From d535bf47d805eeb04a69ec1941296d09cfcd12db Mon Sep 17 00:00:00 2001 From: Jordan Yates Date: Wed, 15 Jul 2026 11:27:40 +1000 Subject: [PATCH 8/8] main_gui: runtime copyright year Compute the current year at runtime instead of buildtime, in order to stop every `cargo run` invocation from triggering a rebuild of every crate in the workspace. Signed-off-by: Jordan Yates --- Cargo.toml | 3 --- build.rs | 8 -------- src/main_gui.rs | 8 ++++---- 3 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 build.rs diff --git a/Cargo.toml b/Cargo.toml index 4d6300d..28c7527 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,5 @@ egui_extras = "0.34.1" directories = "6.0.0" image = "0.25.9" -[build-dependencies] -chrono = "0.4.42" - [target.'cfg(windows)'.dependencies] winapi = { version = "*", features = ["winbase"] } diff --git a/build.rs b/build.rs deleted file mode 100644 index bab8933..0000000 --- a/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -use chrono::{Datelike, Utc}; - -fn main() { - println!( - "cargo:rustc-env=INFUSE_DECODER_BUILD_YEAR={}", - Utc::now().year() - ); -} diff --git a/src/main_gui.rs b/src/main_gui.rs index aa1d1b1..4d145f1 100644 --- a/src/main_gui.rs +++ b/src/main_gui.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::{collections::HashMap, path::PathBuf}; +use chrono::{Datelike, Utc}; use eframe::egui::{self, IconData}; use egui_extras::{Column, TableBuilder}; use image::GenericImageView; @@ -692,11 +693,10 @@ fn copyright_bar(ui: &mut egui::Ui) { .num_columns(2) .show(ui, |ui| { ui.with_layout(egui::Layout::right_to_left(egui::Align::LEFT), |ui| { - ui.label(concat!( - "v", + ui.label(format!( + "v{} © Embeint Inc 2024-{}", env!("CARGO_PKG_VERSION"), - " © Embeint Inc 2024-", - env!("INFUSE_DECODER_BUILD_YEAR") + Utc::now().year() )); });