From e1e153107171cb25ec0134b6bcdb22d139a15719 Mon Sep 17 00:00:00 2001 From: Michael Aebli Date: Thu, 23 Jul 2026 21:17:57 +0200 Subject: [PATCH] Make JSON/YAML output idiomatic with structured record values - Replace the table-style string in summary.records[].value with structured fields (value, exponent, unit, quantity) derived from the processed data record header; keep the human-readable string as 'display' so table and CSV output stay unchanged. - Remove the deprecated top-level manufacturer_info key from JSON/YAML; summary.manufacturer carries the same information. - Serialize raw byte payloads (wireless frame data, record raw_bytes, manufacturer-specific data) as compact uppercase hex strings via serialize-only serde changes. - Bump workspace versions to 0.2.0 and document the migration in the changelog. --- CHANGELOG.md | 19 ++ Cargo.toml | 10 +- cli/Cargo.toml | 4 +- crates/m-bus-application-layer/Cargo.toml | 4 +- .../src/data_information.rs | 8 +- .../src/data_record.rs | 4 + crates/m-bus-core/Cargo.toml | 2 +- crates/m-bus-core/src/lib.rs | 23 ++ crates/wired-mbus-link-layer/Cargo.toml | 4 +- crates/wireless-mbus-link-layer/Cargo.toml | 4 +- crates/wireless-mbus-link-layer/src/lib.rs | 4 + python/Cargo.toml | 4 +- src/mbus_data.rs | 240 +++++++++++++++--- tests/test_other_meters.rs | 2 +- wasm/Cargo.toml | 4 +- 15 files changed, 285 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d1672..492419c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] + +Breaking changes for JSON/YAML consumers: + +- Changed `summary.records[].value` from a table-style string (e.g. + `"(2850427)e-2[m³](Volume)"`) to structured fields: `value` (number), + `exponent`, `unit` and `quantity`, derived from the processed data record + header. The human-readable string is still available as + `summary.records[].display`. Table and CSV outputs are unchanged. +- Removed the top-level `manufacturer_info` key from JSON and YAML output. + Migration: use `summary.manufacturer`, which carries `code`, `name`, + `website` and `description`. +- Changed raw byte payloads (`frame.data` of wireless frames, + `data_records[].raw_bytes` and manufacturer-specific record data) to + serialize as compact uppercase hex strings instead of decimal byte arrays. + Migration: decode the hex string instead of reading a JSON/YAML array + (e.g. `"2F2F"` instead of `[47, 47]`). Serialize-only: parsing APIs and + `Deserialize` implementations are unchanged. + ## [0.1.4] - 2026-07-17 - Added direct application-layer parsing APIs, record accessors, and a crate-local data-record example. diff --git a/Cargo.toml b/Cargo.toml index 4e8ccdd..120451d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "m-bus-parser" -version = "0.1.4" +version = "0.2.0" edition = "2021" description = "A library for parsing M-Bus frames" license = "MIT" @@ -47,10 +47,10 @@ serde = { version = "1.0", features = ["derive"], optional = true } bitflags = "2.8.0" arrayvec = { version = "0.7.4", default-features = false } defmt = { version = "1.0.1", optional = true } -wired-mbus-link-layer = { version = "0.1.4", path = "crates/wired-mbus-link-layer" } -wireless-mbus-link-layer = { version = "0.1.4", path = "crates/wireless-mbus-link-layer" } -m-bus-core = { version = "0.1.4", path = "crates/m-bus-core" } -m-bus-application-layer = { version = "0.1.4", path = "crates/m-bus-application-layer" } +wired-mbus-link-layer = { version = "0.2.0", path = "crates/wired-mbus-link-layer" } +wireless-mbus-link-layer = { version = "0.2.0", path = "crates/wireless-mbus-link-layer" } +m-bus-core = { version = "0.2.0", path = "crates/m-bus-core" } +m-bus-application-layer = { version = "0.2.0", path = "crates/m-bus-application-layer" } aes = { version = "0.9", optional = true, default-features = false } cbc = { version = "0.2", optional = true, default-features = false } cipher = { version = "0.5", optional = true, default-features = false, features = ["block-padding"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index e42cb8d..8d52dbb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "m-bus-parser-cli" -version = "0.1.4" +version = "0.2.0" edition = "2021" description = "A cli to use the library for parsing M-Bus frames" license = "MIT" @@ -22,6 +22,6 @@ default = ["decryption"] decryption = ["m-bus-parser/decryption"] [dependencies] -m-bus-parser = { path = "..", version = "0.1.4", features = ["std", "serde"] } +m-bus-parser = { path = "..", version = "0.2.0", features = ["std", "serde"] } hex = "0.4" clap = { version = "4.5.4", features = ["derive"] } diff --git a/crates/m-bus-application-layer/Cargo.toml b/crates/m-bus-application-layer/Cargo.toml index 265545f..a8dc422 100644 --- a/crates/m-bus-application-layer/Cargo.toml +++ b/crates/m-bus-application-layer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "m-bus-application-layer" -version = "0.1.4" +version = "0.2.0" edition = "2021" description = "M-Bus application layer parser (DIF/VIF, data records)" license = "MIT" @@ -20,4 +20,4 @@ serde = { version = "1.0", features = ["derive"], optional = true } defmt = { version = "1.0.1", optional = true } bitflags = "2.8.0" arrayvec = { version = "0.7.4", default-features = false } -m-bus-core = { version = "0.1.4", path = "../m-bus-core" } +m-bus-core = { version = "0.2.0", path = "../m-bus-core" } diff --git a/crates/m-bus-application-layer/src/data_information.rs b/crates/m-bus-application-layer/src/data_information.rs index 71012ec..683e306 100644 --- a/crates/m-bus-application-layer/src/data_information.rs +++ b/crates/m-bus-application-layer/src/data_information.rs @@ -400,7 +400,13 @@ pub enum DataType<'a> { SingleEveryOrInvalid, SingleEveryOrInvalid, ), - ManufacturerSpecific(&'a [u8]), + ManufacturerSpecific( + #[cfg_attr( + feature = "serde", + serde(serialize_with = "m_bus_core::serde_hex::serialize") + )] + &'a [u8], + ), } #[cfg_attr(feature = "serde", derive(serde::Serialize))] #[derive(PartialEq, Debug, Clone)] diff --git a/crates/m-bus-application-layer/src/data_record.rs b/crates/m-bus-application-layer/src/data_record.rs index d7e6bce..7c005ca 100644 --- a/crates/m-bus-application-layer/src/data_record.rs +++ b/crates/m-bus-application-layer/src/data_record.rs @@ -25,6 +25,10 @@ pub struct DataRecord<'a> { pub data_record_header: DataRecordHeader<'a>, pub data: Data<'a>, /// Raw bytes encompassing this data record + #[cfg_attr( + feature = "serde", + serde(serialize_with = "m_bus_core::serde_hex::serialize") + )] pub raw_bytes: &'a [u8], } diff --git a/crates/m-bus-core/Cargo.toml b/crates/m-bus-core/Cargo.toml index c240d83..a5eb350 100644 --- a/crates/m-bus-core/Cargo.toml +++ b/crates/m-bus-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "m-bus-core" -version = "0.1.4" +version = "0.2.0" edition = "2024" description = "Core types for the m-bus-parser M-Bus protocol library" license = "MIT" diff --git a/crates/m-bus-core/src/lib.rs b/crates/m-bus-core/src/lib.rs index 326cc4c..188fc8d 100644 --- a/crates/m-bus-core/src/lib.rs +++ b/crates/m-bus-core/src/lib.rs @@ -1,5 +1,28 @@ pub mod decryption; +/// Serializes raw byte payloads as compact uppercase hex strings so that +/// human-facing dumps (JSON/YAML) don't render them as decimal byte arrays. +/// Serialize-only: deserialization of these fields is unaffected. +#[cfg(feature = "serde")] +pub mod serde_hex { + use core::fmt; + + struct HexSlice<'a>(&'a [u8]); + + impl fmt::Display for HexSlice<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(f, "{:02X}", byte)?; + } + Ok(()) + } + } + + pub fn serialize(data: &[u8], serializer: S) -> Result { + serializer.collect_str(&HexSlice(data)) + } +} + #[cfg(feature = "std")] use std::fmt::{self, Display}; diff --git a/crates/wired-mbus-link-layer/Cargo.toml b/crates/wired-mbus-link-layer/Cargo.toml index 6564b34..7f60818 100644 --- a/crates/wired-mbus-link-layer/Cargo.toml +++ b/crates/wired-mbus-link-layer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wired-mbus-link-layer" -version = "0.1.4" +version = "0.2.0" edition = "2024" description = "Wired M-Bus link layer parser (EN 13757-2)" license = "MIT" @@ -15,4 +15,4 @@ defmt = ["dep:defmt"] [dependencies] serde = { version = "1.0", features = ["derive"], optional = true } defmt = { version = "1.0.1", optional = true } -m-bus-core = { version = "0.1.4", path = "../m-bus-core" } +m-bus-core = { version = "0.2.0", path = "../m-bus-core" } diff --git a/crates/wireless-mbus-link-layer/Cargo.toml b/crates/wireless-mbus-link-layer/Cargo.toml index cd37b17..5e903da 100644 --- a/crates/wireless-mbus-link-layer/Cargo.toml +++ b/crates/wireless-mbus-link-layer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wireless-mbus-link-layer" -version = "0.1.4" +version = "0.2.0" edition = "2024" description = "Wireless M-Bus link layer parser (wMBus)" license = "MIT" @@ -14,6 +14,6 @@ defmt = ["dep:defmt"] [dependencies] crc16 = "0.4.0" -m-bus-core = { version = "0.1.4", path = "../m-bus-core" } +m-bus-core = { version = "0.2.0", path = "../m-bus-core" } serde = { version = "1.0", features = ["derive"], optional = true } defmt = { version = "1.0.1", optional = true } diff --git a/crates/wireless-mbus-link-layer/src/lib.rs b/crates/wireless-mbus-link-layer/src/lib.rs index 0edb8ae..c2a2ee8 100644 --- a/crates/wireless-mbus-link-layer/src/lib.rs +++ b/crates/wireless-mbus-link-layer/src/lib.rs @@ -96,6 +96,10 @@ pub fn strip_format_a_crcs<'a>(data: &[u8], output: &'a mut [u8]) -> Option<&'a pub struct WirelessFrame<'a> { pub function: Function, pub manufacturer_id: ManufacturerId, + #[cfg_attr( + feature = "serde", + serde(serialize_with = "m_bus_core::serde_hex::serialize") + )] pub data: &'a [u8], } diff --git a/python/Cargo.toml b/python/Cargo.toml index f78ab91..d258941 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pymbusparser" -version = "0.1.4" +version = "0.2.0" edition = "2021" homepage = "https://maebli.github.io/" repository = "https://github.com/maebli/m-bus-parser" @@ -8,7 +8,7 @@ description = "A Python binding for the M-Bus parser" license = "MIT" [dependencies] -m-bus-parser = { path = "..", version = "0.1.4", features = ["std", "serde", "decryption"] } +m-bus-parser = { path = "..", version = "0.2.0", features = ["std", "serde", "decryption"] } serde_json = "1.0" pyo3 = { version = "0.29.0", features = ["extension-module", "generate-import-lib"] } hex = "0.4" diff --git a/src/mbus_data.rs b/src/mbus_data.rs index d0a2e7d..120accc 100644 --- a/src/mbus_data.rs +++ b/src/mbus_data.rs @@ -187,7 +187,16 @@ struct ManufacturerSummary { #[cfg(feature = "std")] #[derive(serde::Serialize)] struct RecordSummary { - value: String, + /// Human-readable rendering (same string the table and CSV outputs use). + display: String, + #[serde(skip_serializing_if = "Option::is_none")] + value: Option, + #[serde(skip_serializing_if = "Option::is_none")] + exponent: Option, + #[serde(skip_serializing_if = "Option::is_none")] + unit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + quantity: Option, data_information: String, header_hex: String, data_hex: String, @@ -280,16 +289,19 @@ fn variable_data_block<'a>(user_data: Option<&user_data::UserDataBlock<'a>>) -> #[cfg(feature = "std")] fn record_summaries(records: &user_data::DataRecords) -> Vec { + use user_data::data_information::DataType; + records .clone() .flatten() .map(|record| { - let value_information = match record + let value_information = record .data_record_header .processed_data_record_header .value_information - { - Some(ref x) => format!("{}", x), + .as_ref(); + let value_information_display = match value_information { + Some(x) => format!("{}", x), None => ")".to_string(), }; let data_information = match record @@ -300,8 +312,28 @@ fn record_summaries(records: &user_data::DataRecords) -> Vec { Some(ref x) => format!("{}", x), None => "None".to_string(), }; + let value = match record.data.value { + Some(DataType::Number(n)) | Some(DataType::LossyNumber(n)) => Some(n), + _ => None, + }; + let unit = value_information + .map(|vi| vi.units.iter().map(ToString::to_string).collect::()) + .filter(|s| !s.is_empty()); + let quantity = value_information + .map(|vi| { + vi.labels + .iter() + .map(|label| format!("{:?}", label)) + .collect::>() + .join(", ") + }) + .filter(|s| !s.is_empty()); RecordSummary { - value: format!("({}{}", record.data, value_information), + display: format!("({}{}", record.data, value_information_display), + value, + exponent: value_information.map(|vi| vi.decimal_scale_exponent), + unit, + quantity, data_information, header_hex: record.data_record_header_hex(), data_hex: record.data_hex(), @@ -450,20 +482,6 @@ fn summarize_wireless( fn build_output(parsed: &T, summary: FrameSummary) -> ParsedOutput { let mut json = serde_json::to_value(parsed).unwrap_or_default(); if let serde_json::Value::Object(ref mut map) = json { - if let Some(mfr) = &summary.manufacturer { - if let (Some(name), Some(website), Some(description)) = - (&mfr.name, &mfr.website, &mfr.description) - { - map.insert( - "manufacturer_info".to_string(), - serde_json::json!({ - "name": name, - "website": website, - "description": description, - }), - ); - } - } map.insert( "summary".to_string(), serde_json::to_value(&summary).unwrap_or_default(), @@ -471,16 +489,6 @@ fn build_output(parsed: &T, summary: FrameSummary) -> Parse } let mut yaml = serde_yaml::to_string(parsed).unwrap_or_default(); - if let Some(mfr) = &summary.manufacturer { - if let (Some(name), Some(website), Some(description)) = - (&mfr.name, &mfr.website, &mfr.description) - { - yaml.push_str(&format!( - "manufacturer_info:\n name: {}\n website: {}\n description: {}\n", - name, website, description - )); - } - } yaml.push_str( &serde_yaml::to_string(&SummarySection { summary: &summary }).unwrap_or_default(), ); @@ -694,7 +702,7 @@ fn render_table(summary: &FrameSummary, key_provided: bool) -> String { value_table.set_titles(row!["Value", "Data Information", "Header Hex", "Data Hex"]); for record in &summary.records { value_table.add_row(row![ - record.value, + record.display, record.data_information, record.header_hex, record.data_hex @@ -776,7 +784,7 @@ pub fn parse_to_csv(input: &str, key: Option<&[u8; 16]>) -> String { summary.device_type.clone().unwrap_or_default(), ]; for record in &summary.records { - row.push(record.value.clone()); + row.push(record.display.clone()); row.push(record.data_information.clone()); } writer @@ -1535,9 +1543,179 @@ mod tests { ); } } + + // The structured record fields in the JSON summary must agree with + // the normalized summary the table and CSV render from. + let parsed: serde_json::Value = + serde_json::from_str(&json).expect("json output should be valid"); + let json_records = parsed + .get("summary") + .and_then(|s| s.get("records")) + .and_then(|r| r.as_array()) + .expect("json summary should contain records"); + assert_eq!(json_records.len(), summary.records.len()); + for (json_record, record) in json_records.iter().zip(&summary.records) { + assert_eq!( + json_record.get("display").and_then(|v| v.as_str()), + Some(record.display.as_str()), + "json record display mismatch for input {input}" + ); + assert_eq!( + json_record.get("value").and_then(|v| v.as_f64()), + record.value, + "json record value mismatch for input {input}" + ); + assert_eq!( + json_record + .get("exponent") + .and_then(|v| v.as_i64()) + .map(|v| v as isize), + record.exponent, + "json record exponent mismatch for input {input}" + ); + assert_eq!( + json_record.get("unit").and_then(|v| v.as_str()), + record.unit.as_deref(), + "json record unit mismatch for input {input}" + ); + assert_eq!( + json_record.get("quantity").and_then(|v| v.as_str()), + record.quantity.as_deref(), + "json record quantity mismatch for input {input}" + ); + // The display string the table/CSV formats use must embed the + // same structured information. + if let Some(unit) = &record.unit { + assert!( + record.display.contains(unit.as_str()), + "display {:?} should contain unit {unit:?}", + record.display + ); + } + if let Some(exponent) = record.exponent.filter(|e| *e != 0) { + assert!( + record.display.contains(&format!("e{exponent}")), + "display {:?} should contain exponent {exponent}", + record.display + ); + } + } } } + #[cfg(feature = "std")] + #[test] + fn test_json_records_contain_structured_values() { + let input = "68 3D 3D 68 08 01 72 00 51 20 02 82 4D 02 04 00 88 00 00 04 07 00 00 00 00 0C 15 03 00 00 00 0B 2E 00 00 00 0B 3B 00 00 00 0A 5A 88 12 0A 5E 16 05 0B 61 23 77 00 02 6C 8C 11 02 27 37 0D 0F 60 00 67 16"; + let json_output = super::parse_to_json(input, None); + let parsed: serde_json::Value = + serde_json::from_str(&json_output).expect("json output should be valid"); + let records = parsed + .get("summary") + .and_then(|s| s.get("records")) + .and_then(|r| r.as_array()) + .expect("summary should contain records"); + + // "(3)e-1[m³](Volume)" becomes structured fields. + let volume = records + .iter() + .find(|r| r.get("quantity").and_then(|v| v.as_str()) == Some("Volume")) + .expect("volume record"); + assert_eq!(volume.get("value").and_then(|v| v.as_f64()), Some(3.0)); + assert_eq!(volume.get("exponent").and_then(|v| v.as_i64()), Some(-1)); + assert_eq!(volume.get("unit").and_then(|v| v.as_str()), Some("m³")); + assert_eq!( + volume.get("display").and_then(|v| v.as_str()), + Some("(3)e-1[m³](Volume)") + ); + + // A date record has no numeric value but keeps its display string. + let date = records + .iter() + .find(|r| r.get("quantity").and_then(|v| v.as_str()) == Some("Date")) + .expect("date record"); + assert!(date.get("value").is_none()); + assert_eq!( + date.get("display").and_then(|v| v.as_str()), + Some("(12/Jan/12)(Date)") + ); + + // A manufacturer-specific record has neither value information nor a + // numeric value. + let manufacturer_specific = records + .iter() + .find(|r| { + r.get("data_information") + .and_then(|v| v.as_str()) + .is_some_and(|s| s.contains("ManufacturerSpecific")) + }) + .expect("manufacturer specific record"); + assert!(manufacturer_specific.get("value").is_none()); + assert!(manufacturer_specific.get("exponent").is_none()); + assert!(manufacturer_specific.get("unit").is_none()); + assert!(manufacturer_specific.get("quantity").is_none()); + } + + #[cfg(feature = "std")] + #[test] + fn test_json_and_yaml_have_no_manufacturer_info() { + let input = "68 3D 3D 68 08 01 72 00 51 20 02 82 4D 02 04 00 88 00 00 04 07 00 00 00 00 0C 15 03 00 00 00 0B 2E 00 00 00 0B 3B 00 00 00 0A 5A 88 12 0A 5E 16 05 0B 61 23 77 00 02 6C 8C 11 02 27 37 0D 0F 60 00 67 16"; + let json_output = super::parse_to_json(input, None); + let parsed: serde_json::Value = + serde_json::from_str(&json_output).expect("json output should be valid"); + assert!(parsed.get("manufacturer_info").is_none()); + // summary.manufacturer carries the same information instead. + assert_eq!( + parsed + .get("summary") + .and_then(|s| s.get("manufacturer")) + .and_then(|m| m.get("name")) + .and_then(|v| v.as_str()), + Some("Schlumberger Industries") + ); + + let yaml_output = super::parse_to_yaml(input, None); + assert!(!yaml_output.contains("manufacturer_info:")); + assert!(yaml_output.contains("name: Schlumberger Industries")); + } + + #[cfg(feature = "std")] + #[test] + fn test_byte_payloads_serialize_as_hex_strings() { + // Wireless frame: frame.data must be a compact uppercase hex string. + let wireless_input = "1444AE0C7856341201078C2027780B134365877AC5"; + let json_output = super::parse_to_json(wireless_input, None); + let parsed: serde_json::Value = + serde_json::from_str(&json_output).expect("json output should be valid"); + let frame_data = parsed + .get("frame") + .and_then(|f| f.get("data")) + .expect("wireless frame should contain data"); + let hex = frame_data.as_str().expect("frame data should be a string"); + assert!(!hex.is_empty()); + assert!(hex.chars().all(|c| c.is_ascii_hexdigit())); + assert_eq!(hex, hex.to_uppercase()); + + // Data records: raw_bytes must be hex strings as well. + let wired_input = "68 3D 3D 68 08 01 72 00 51 20 02 82 4D 02 04 00 88 00 00 04 07 00 00 00 00 0C 15 03 00 00 00 0B 2E 00 00 00 0B 3B 00 00 00 0A 5A 88 12 0A 5E 16 05 0B 61 23 77 00 02 6C 8C 11 02 27 37 0D 0F 60 00 67 16"; + let json_output = super::parse_to_json(wired_input, None); + let parsed: serde_json::Value = + serde_json::from_str(&json_output).expect("json output should be valid"); + let records = parsed + .get("data_records") + .and_then(|r| r.as_array()) + .expect("data records"); + let first_raw_bytes = records + .first() + .and_then(|r| r.get("raw_bytes")) + .and_then(|v| v.as_str()) + .expect("raw_bytes should be a hex string"); + assert_eq!(first_raw_bytes, "040700000000"); + + // Manufacturer-specific data is a hex string instead of a byte array. + assert!(json_output.contains("\"ManufacturerSpecific\": \"6000\"")); + } + #[cfg(feature = "std")] #[test] fn test_yaml_expected_output() { diff --git a/tests/test_other_meters.rs b/tests/test_other_meters.rs index 8659137..99aa07c 100644 --- a/tests/test_other_meters.rs +++ b/tests/test_other_meters.rs @@ -130,7 +130,7 @@ mod tests { 0x1F ); assert!(rec["data"]["value"]["ManufacturerSpecific"] - .as_array() + .as_str() .unwrap() .is_empty()); } diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index d188270..402039b 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "m-bus-parser-wasm-pack" -version = "0.1.4" +version = "0.2.0" edition = "2021" description = "A wasm-pack to use the library for parsing M-Bus frames" license = "MIT" @@ -22,7 +22,7 @@ decryption = ["m-bus-parser/decryption"] [dependencies] wasm-bindgen = "0.2.84" -m-bus-parser = { path = "..", version = "0.1.4", features = ["std", "serde"] } +m-bus-parser = { path = "..", version = "0.2.0", features = ["std", "serde"] } serde = { version = "1.0" } serde_json = "1.0"