diff --git a/.gitignore b/.gitignore index 3a24643..704bf14 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /target +**/target/* *.bin *.csv +*.parquet Cargo.lock \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index e148b35..e50fc5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org). +## [1.9.0] - 2026-06-16 + + - New output format [Apache Parquet](https://parquet.apache.org/) + - Option to limit output files to a certain number of readings + * If the limit is hit, output files have a numeric postfix + - Option to skip merge step to optimize decoding times + ## [1.8.1] - 2026-06-05 - MacOS GUI distributed as notorized `.dmg` diff --git a/Cargo.toml b/Cargo.toml index 498cffc..b044fac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "infuse_decoder" -version = "1.8.1" +version = "1.9.0" edition = "2024" [[bin]] @@ -20,6 +20,9 @@ indicatif = "0.18.3" itertools = "0.14.0" memmap = "0.7.0" num_cpus = "1.17.0" +arrow-array = "59.0.0" +arrow-schema = "59.0.0" +parquet = "59.0.0" prettytable-rs = "0.10.0" regex = "1.12.2" rfd = "0.16.0" diff --git a/README.md b/README.md index 83f2d60..8d5c45c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ The application provides options to control the input selection and output gener ### 1) Output Folder -This field controls where the output data will be placed after decoding. It defaults to $USER_HOME/infuse_iot. The output folder can be updated by clicking `Folder` (1a). +This field controls where the output data will be placed after decoding. It defaults to $USER_HOME/infuse_iot. The output folder can be updated by clicking `Folder` (1a). The output folder can be opened +in the default system viewer by clicking `Open` (1b). ### 2) Input folder/file @@ -32,13 +33,36 @@ If data from multiple Tauro collars exists on a single SD card, this option will This field controls the prefix of the output filenames, created in the output folder. This value will default to the Device ID. -### 5) Time Output Format +### 5) Output Format + +Save the output as either Comma Separated Value (CSV) or [Apache Parquet](https://parquet.apache.org/) files. +Decoding to Parquet is faster and the resulting files are smaller, but the results are not human readable. + +### 6) Linearize Output + +To optimize processing times, the decoding work is split across multiple CPU cores to intermediate files, with a +post-processing step pulling the intermediate results back into a single file. +If a single output file is not required, disabling this skips the merging step, saving decoding time. +If disabled, the final files have a numeric postfix (e.g. `test_BATTERY_STATE_00000.csv`) which indicates the order from the +original binary file. + +### 7) Maximum Readings per File + +If the Linearize Output step is enabled, the output data can be split into multiple files based on the number of rows in each file. +This can be useful to limit individual files sizes or optimize data loading. The default value of 0 means no limit. + +### 8) Time Output Format This option controls the output format of the timestamps written into the output CSV files. The two options are a [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) formatted string (for example 2024-06-27T13:55:12.123456Z), or a Unix timestamp with subseconds (for example 1731457165.123456). The RFC3339 option is recommended if the CSV outputs will be looked at by users, as it is a more human-readable format. By comparison, the Unix timestamps are simpler for data processing scripts to parse, and are faster for the decoder tool to generate. -### 6) Decode +### 9) Input Block Size + +Specifies the data block size of the input binary data. The default value of `512` should be used unless instructions to the +contrary are provided. + +### 10) Decode Once an input file or folder has been selected, the decode button becomes available to select. Clicking this button begins the decode process with the currently selected options. The button is unavailable to select again until the previous decode has completed. ## Decoding Process @@ -49,11 +73,11 @@ The data on the SD card is copied onto the local filesystem to create the merged ### 2) Decoding files -The binary data format is decoded into human-readable CSV files. This process is run across all CPU cores to maximize performance. +The binary data format is decoded into human-readable CSV or Apache Parquet files. This process is run across all CPU cores to maximize performance. ### 3) Merging output -The individual CSV files created by each thread in the decoding step are merged back together into a single file per sensor stream. +The individual files created by each thread in the decoding step are merged back together into a single file per sensor stream, unless output linearization is disabled. ## Output Statistics diff --git a/assets/configuration_options.png b/assets/configuration_options.png index 086e4e7..0ae3789 100644 Binary files a/assets/configuration_options.png and b/assets/configuration_options.png differ diff --git a/scripts/tdf_decoder_build.py b/scripts/tdf_decoder_build.py index b92230a..f047292 100644 --- a/scripts/tdf_decoder_build.py +++ b/scripts/tdf_decoder_build.py @@ -5,6 +5,7 @@ import json import os import pathlib +import re from numpy import format_float_positional as float_format from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -25,6 +26,143 @@ "float64_t": ("f64", True), } +arrow_int_type = { + "int8_t": "DataType::Int8", + "uint8_t": "DataType::UInt8", + "int16_t": "DataType::Int16", + "uint16_t": "DataType::UInt16", + "int32_t": "DataType::Int32", + "uint32_t": "DataType::UInt32", + "int64_t": "DataType::Int64", + "uint64_t": "DataType::UInt64", +} + +arrow_float_type = { + "float": "DataType::Float32", + "float32_t": "DataType::Float32", + "float64_t": "DataType::Float64", +} + + +def rust_str(value): + return json.dumps(value) + + +def arrow_name(name): + name = re.sub(r"\W", "_", name) + if not name or name[0].isdigit(): + name = f"_{name}" + return name + + +def indent_block(value, indent): + pad = " " * indent + return "\n".join(f"{pad}{line}" for line in value.splitlines()) + + +def arrow_scalar_type(field): + c_type = field["type"] + conv = field.get("conversion", {}) + + if c_type == "char": + return "DataType::Utf8" + + if "m" in conv or "c" in conv: + return "DataType::Float64" + + if "int" in conv: + assert "num" in field + byte_len = field["num"] + if byte_len <= 1: + return "DataType::UInt8" + if byte_len <= 2: + return "DataType::UInt16" + if byte_len <= 4: + return "DataType::UInt32" + if byte_len <= 8: + return "DataType::UInt64" + return f"DataType::FixedSizeBinary({byte_len})" + + if c_type in arrow_int_type: + return arrow_int_type[c_type] + + if c_type in arrow_float_type: + return arrow_float_type[c_type] + + raise RuntimeError(f"Bad type '{c_type}'") + + +def arrow_data_type_expr(field, structs, indent): + c_type = field["type"] + conv = field.get("conversion", {}) + + if c_type.startswith("struct "): + struct_name = c_type.removeprefix("struct ") + if struct_name not in structs: + raise RuntimeError(f"Bad type '{c_type}'") + nested = ",\n".join( + arrow_field_expr(nested, structs, indent + 8) + for nested in structs[struct_name]["fields"] + ) + base = "DataType::Struct(Fields::from(vec![\n" + base += f"{indent_block(nested, indent + 4)}\n" + base += f"{' ' * indent}]))" + else: + base = arrow_scalar_type(field) + + if "num" not in field or field["type"] == "char" or "int" in conv: + return base + + num = field["num"] + if num == 0: + if c_type == "uint8_t": + return "DataType::Binary" + return ( + "DataType::List(Arc::new(Field::new_list_field(\n" + f"{indent_block(base, indent + 4)},\n" + f"{' ' * (indent + 4)}false,\n" + f"{' ' * indent})))" + ) + + if c_type == "uint8_t": + return f"DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::UInt8, false)), {num})" + return ( + "DataType::FixedSizeList(Arc::new(Field::new_list_field(\n" + f"{indent_block(base, indent + 4)},\n" + f"{' ' * (indent + 4)}false,\n" + f"{' ' * indent})), " + f"{num})" + ) + + +def arrow_field_expr(field, structs, indent): + field_name = rust_str(arrow_name(field["name"])) + data_type = arrow_data_type_expr(field, structs, indent + 4) + if "\n" not in data_type: + return f"Field::new({field_name}, {data_type}, false)" + return ( + "Field::new(\n" + f"{' ' * (indent + 4)}{field_name},\n" + f"{indent_block(data_type, indent + 4)},\n" + f"{' ' * (indent + 4)}false,\n" + f"{' ' * indent})" + ) + + +def arrow_schema_expr(info, structs): + fields = ",\n".join( + arrow_field_expr(field, structs, 12) for field in info["fields"] + ) + return ( + "Schema::new(vec![\n" + + indent_block("timestamp_field(),", 8) + + "\n" + + indent_block("sample_idx_field(),", 8) + + "\n" + + indent_block(fields, 8) + + "\n ])" + ) + def decoders_gen(tdf_defs, output): env = Environment( @@ -35,6 +173,10 @@ def decoders_gen(tdf_defs, output): ) common_template = env.get_template("tdf_decoder.rs.jinja") csv_template = env.get_template("tdf_decoder_csv.rs.jinja") + parquet_template = env.get_template("tdf_decoder_parquet.rs.jinja") + + for _tdf_id, info in tdf_defs["definitions"].items(): + info["arrow_schema"] = arrow_schema_expr(info, tdf_defs["structs"]) def field_conv_func(field, name_prefix=None): t = rust_type[field["type"]] @@ -42,8 +184,10 @@ def field_conv_func(field, name_prefix=None): if t[1]: func += "::" func += "()?" + int_endian = None if c := field.get("conversion"): - if endian := c.get("int", None): + int_endian = c.get("int", None) + if endian := int_endian: assert "num" in field assert t[0] == "u8" e = "LittleEndian" if endian == "little" else "BigEndian" @@ -55,7 +199,6 @@ def field_conv_func(field, name_prefix=None): raise RuntimeError("Unknown integer length") func = f"cursor.read_{t}::<{e}>()?" - del field["num"] if "m" in c or "c" in c: func += " as f64" @@ -76,7 +219,7 @@ def field_conv_func(field, name_prefix=None): if name_prefix is not None: n = f"{name_prefix}." + n - if "num" in field: + if "num" in field and not int_endian: if field["type"] == "char": return [ ( @@ -104,6 +247,8 @@ def field_fmt(field): single = ["0x{:x}"] else: single = ["{}"] + if field.get("conversion", {}).get("int", None): + return single if field.get("num", None) == 0: return ["{}"] return single * field.get("num", 1) @@ -136,21 +281,383 @@ def field_fmt(field): info["rust_head"] = ",".join([f'"{c[0]}"' for c in info["rust_convs"]]) info["rust_fmt"] = ",".join(fmt) + rust_array_type = { + "i8": "Int8Array", + "u8": "UInt8Array", + "i16": "Int16Array", + "u16": "UInt16Array", + "i32": "Int32Array", + "u32": "UInt32Array", + "i64": "Int64Array", + "u64": "UInt64Array", + "f32": "Float32Array", + "f64": "Float64Array", + } + + rust_vec_type = { + "char": "String", + "int8_t": "i8", + "uint8_t": "u8", + "int16_t": "i16", + "uint16_t": "u16", + "int32_t": "i32", + "uint32_t": "u32", + "int64_t": "i64", + "uint64_t": "u64", + "float": "f32", + "float32_t": "f32", + "float64_t": "f64", + } + + def rust_type_after_conversion(field): + conv = field.get("conversion", {}) + if "m" in conv or "c" in conv: + return "f64" + if field["type"] == "char": + return "String" + if "int" in conv: + byte_len = field["num"] + if byte_len <= 1: + return "u8" + if byte_len <= 2: + return "u16" + if byte_len <= 4: + return "u32" + if byte_len <= 8: + return "u64" + return rust_vec_type[field["type"]] + + def rust_pascal(value): + return "".join(part.capitalize() for part in re.split(r"\W|_", value) if part) + + def rust_field_ident(path): + return arrow_name("_".join(path)) + + def primitive_read_expr(field): + t = rust_type[field["type"]] + func = f"cursor.read_{t[0]}" + if t[1]: + func += "::" + func += "()?" + + if c := field.get("conversion"): + if endian := c.get("int", None): + e = "LittleEndian" if endian == "little" else "BigEndian" + if field["num"] == 3: + t_name = "u24" + elif field["num"] == 6: + t_name = "u48" + else: + raise RuntimeError("Unknown integer length") + func = f"cursor.read_{t_name}::<{e}>()?" + + if "m" in c or "c" in c: + func += " as f64" + if "m" in c and c["m"] != 0: + val = c["m"] + inverse_ratio = (1 / val).as_integer_ratio() + if inverse_ratio[1] == 1: + func += f" / {inverse_ratio[0]}.0" + else: + func += f" * {float_format(c['m'])}" + if "c" in c and c["c"] != 0: + func += f" + {float_format(c['c'])}" + return func + + def field_byte_size(field): + c_type = field["type"] + conv = field.get("conversion", {}) + if "int" in conv: + return field["num"] + if c_type.startswith("struct "): + struct_name = c_type.removeprefix("struct ") + return sum(field_byte_size(f) for f in tdf_defs["structs"][struct_name]["fields"]) + if c_type == "char": + return field.get("num", 0) + base = { + "int8_t": 1, + "uint8_t": 1, + "int16_t": 2, + "uint16_t": 2, + "int32_t": 4, + "uint32_t": 4, + "int64_t": 8, + "uint64_t": 8, + "float": 4, + "float32_t": 4, + "float64_t": 8, + }[c_type] + return base * field.get("num", 1) + + def field_model(field, path): + c_type = field["type"] + num = field.get("num", None) + conv = field.get("conversion", {}) + + if c_type.startswith("struct "): + struct_name = c_type.removeprefix("struct ") + child_fields = tdf_defs["structs"][struct_name]["fields"] + children = [field_model(child, path + [arrow_name(child["name"])]) for child in child_fields] + if num == 0: + return { + "kind": "list", + "path": path, + "child": {"kind": "struct", "path": path, "children": children}, + "item_size": field_byte_size(field), + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + } + return { + "kind": "struct", + "path": path, + "children": children, + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + } + + if c_type == "char": + return { + "kind": "string", + "path": path, + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + "read": f"tdf_field_read_string_to_string(cursor, cursor_start, {num or 0}, size)?", + } + + if num == 0 and c_type == "uint8_t" and "int" not in conv: + return { + "kind": "binary", + "path": path, + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + } + + if num == 0: + return { + "kind": "list", + "path": path, + "child": field_model({k: v for k, v in field.items() if k != "num"}, path), + "item_size": field_byte_size({k: v for k, v in field.items() if k != "num"}), + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + } + + if num is not None and "int" not in conv: + return { + "kind": "fixed_list", + "path": path, + "child": field_model({k: v for k, v in field.items() if k != "num"}, path), + "num": num, + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + } + + rust_type_name = rust_type_after_conversion(field) + return { + "kind": "primitive", + "path": path, + "rust_type": rust_type_name, + "array_type": rust_array_type[rust_type_name], + "field_expr": arrow_field_expr(field, tdf_defs["structs"], 12), + "read": primitive_read_expr(field), + } + + def model_storage_fields(model, out): + kind = model["kind"] + if kind == "primitive": + out.append((rust_field_ident(model["path"]), f"Vec<{model['rust_type']}>")) + elif kind == "string": + out.append((rust_field_ident(model["path"]), "Vec")) + elif kind == "binary": + out.append((rust_field_ident(model["path"]), "Vec>")) + elif kind == "fixed_list": + model_storage_fields(model["child"], out) + elif kind == "list": + out.append((rust_field_ident(model["path"]) + "_offsets", "Vec")) + model_storage_fields(model["child"], out) + elif kind == "struct": + for child in model["children"]: + model_storage_fields(child, out) + else: + raise RuntimeError(f"Bad model kind {kind}") + + def model_init_fields(model, out, capacity): + kind = model["kind"] + if kind == "primitive": + out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + elif kind == "string": + out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + elif kind == "binary": + out.append(f"{rust_field_ident(model['path'])}: Vec::with_capacity({capacity})") + elif kind == "fixed_list": + model_init_fields(model["child"], out, f"{capacity} * {model['num']}") + elif kind == "list": + out.append(f"{rust_field_ident(model['path'])}_offsets: vec![0]") + model_init_fields(model["child"], out, capacity) + elif kind == "struct": + for child in model["children"]: + model_init_fields(child, out, capacity) + else: + raise RuntimeError(f"Bad model kind {kind}") + + def model_append_lines(model, out, cursor_name="cursor"): + kind = model["kind"] + if kind == "primitive": + out.append(f"self.{rust_field_ident(model['path'])}.push({model['read']});") + elif kind == "string": + out.append(f"self.{rust_field_ident(model['path'])}.push({model['read']});") + elif kind == "binary": + out.append( + f"self.{rust_field_ident(model['path'])}.push(crate::decoders::tdf_field_read_vla({cursor_name}, cursor_start, size)?);" + ) + elif kind == "fixed_list": + for _ in range(model["num"]): + model_append_lines(model["child"], out, cursor_name) + elif kind == "list": + out.append("{") + out.append( + f" let bytes_remaining = crate::decoders::vla_bytes_remaining({cursor_name}, cursor_start, size)?;" + ) + out.append(f" if bytes_remaining % {model['item_size']} != 0 {{") + out.append( + ' return Err(Error::new(ErrorKind::InvalidData, "Variable-length array does not align to element size"));' + ) + out.append(" }") + out.append(f" let item_count = bytes_remaining / {model['item_size']};") + out.append(" for _ in 0..item_count {") + child_lines = [] + model_append_lines(model["child"], child_lines, cursor_name) + out.extend([f" {line}" for line in child_lines]) + out.append(" }") + out.append( + f" self.{rust_field_ident(model['path'])}_offsets.push(*self.{rust_field_ident(model['path'])}_offsets.last().unwrap() + item_count as i32);" + ) + out.append("}") + elif kind == "struct": + for child in model["children"]: + model_append_lines(child, out, cursor_name) + else: + raise RuntimeError(f"Bad model kind {kind}") + + def model_finish_expr(model): + kind = model["kind"] + if kind == "primitive": + return f"Arc::new({model['array_type']}::from(std::mem::take(&mut self.{rust_field_ident(model['path'])}))) as ArrayRef" + if kind == "string": + return f"Arc::new(StringArray::from_iter_values(std::mem::take(&mut self.{rust_field_ident(model['path'])}))) as ArrayRef" + if kind == "binary": + return f"Arc::new(BinaryArray::from_iter_values(std::mem::take(&mut self.{rust_field_ident(model['path'])}))) as ArrayRef" + if kind == "fixed_list": + child = model_finish_expr(model["child"]) + child_field = arrow_data_type_expr( + {k: v for k, v in model.get("raw_field", {}).items() if k != "num"}, + tdf_defs["structs"], + 16, + ) + return ( + "Arc::new(FixedSizeListArray::try_new(\n" + f" Arc::new(Field::new_list_field({child_field}, false)),\n" + f" {model['num']},\n" + f" {child},\n" + " None,\n" + " )?) as ArrayRef" + ) + if kind == "list": + child = model_finish_expr(model["child"]) + offsets = rust_field_ident(model["path"]) + "_offsets" + return ( + "{\n" + f" let offsets = std::mem::replace(&mut self.{offsets}, vec![0]);\n" + f" Arc::new(ListArray::try_new(\n" + f" self.list_value_field({model['field_index']}),\n" + " OffsetBuffer::new(ScalarBuffer::from(offsets)),\n" + f" {child},\n" + " None,\n" + " )?) as ArrayRef\n" + " }" + ) + if kind == "struct": + child_fields = ",\n ".join(child["field_expr"] for child in model["children"]) + child_arrays = ",\n ".join(model_finish_expr(child) for child in model["children"]) + return ( + "Arc::new(StructArray::try_new(\n" + f" Fields::from(vec![\n {child_fields}\n ]),\n" + f" vec![\n {child_arrays}\n ],\n" + " None,\n" + " )?) as ArrayRef" + ) + raise RuntimeError(f"Bad model kind {kind}") + + def model_set_raw_field(model, field): + model["raw_field"] = field + if model["kind"] in ("fixed_list", "list"): + child = {k: v for k, v in field.items() if k != "num"} + model_set_raw_field(model["child"], child) + + def model_has_list(model): + if model["kind"] == "list": + return True + if model["kind"] == "fixed_list": + return model_has_list(model["child"]) + if model["kind"] == "struct": + return any(model_has_list(child) for child in model["children"]) + return False + + for tdf_id, info in tdf_defs["definitions"].items(): + info["rust_builder_name"] = f"Tdf{tdf_id}{rust_pascal(info['name'])}Builder" + info["rust_variant_name"] = f"Tdf{tdf_id}{rust_pascal(info['name'])}" + info["parquet_fields"] = [] + models = [] + for idx, field in enumerate(info["fields"]): + model = field_model(field, [arrow_name(field["name"])]) + model["field_index"] = idx + 2 + model_set_raw_field(model, field) + models.append(model) + info["parquet_models"] = models + info["parquet_has_lists"] = any(model_has_list(model) for model in models) + + storage_fields = [ + ("row_timestamp", "Vec>"), + ("row_sample_idx", "Vec>"), + ] + for model in models: + model_storage_fields(model, storage_fields) + info["parquet_storage_fields"] = storage_fields + + init_fields = [ + "row_timestamp: Vec::with_capacity(capacity)", + "row_sample_idx: Vec::with_capacity(capacity)", + ] + for model in models: + model_init_fields(model, init_fields, "capacity") + info["parquet_init_fields"] = init_fields + + append_lines = [ + "self.row_timestamp.push(meta.time_unix_micros);", + "self.row_sample_idx.push(meta.sample_idx);", + ] + for model in models: + model_append_lines(model, append_lines) + info["parquet_append_lines"] = append_lines + + finish_arrays = [ + 'Arc::new(TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)).with_timezone("+00:00")) as ArrayRef', + "Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef", + ] + for idx, model in enumerate(models): + model["field_index"] = idx + 2 + finish_arrays.append(model_finish_expr(model)) + info["parquet_finish_arrays"] = finish_arrays + common_output = pathlib.Path(output) / "decoders.rs" csv_output = pathlib.Path(output) / "decoders_csv.rs" - with common_output.open("w") as f: - rendered = common_template.render( - structs=tdf_defs["structs"], definitions=tdf_defs["definitions"] - ) - f.write(rendered) - f.write(os.linesep) + parquet_output = pathlib.Path(output) / "decoders_parquet.rs" - with csv_output.open("w") as f: - rendered = csv_template.render( - structs=tdf_defs["structs"], definitions=tdf_defs["definitions"] - ) - f.write(rendered) - f.write(os.linesep) + def write_rendered(path, template): + with path.open("w", newline="\n") as f: + rendered = template.render( + structs=tdf_defs["structs"], definitions=tdf_defs["definitions"] + ) + f.write(rendered) + f.write(os.linesep) + + write_rendered(common_output, common_template) + write_rendered(csv_output, csv_template) + write_rendered(parquet_output, parquet_template) if __name__ == "__main__": diff --git a/scripts/tdf_decoder_parquet.rs.jinja b/scripts/tdf_decoder_parquet.rs.jinja new file mode 100644 index 0000000..18df8c6 --- /dev/null +++ b/scripts/tdf_decoder_parquet.rs.jinja @@ -0,0 +1,212 @@ +use std::io::{Cursor, Error, ErrorKind, Read, Result}; +use std::sync::Arc; + +use arrow_array::{ + ArrayRef, BinaryArray, FixedSizeListArray, Float32Array, Float64Array, Int8Array, Int16Array, + Int32Array, ListArray, RecordBatch, StringArray, StructArray, + TimestampMicrosecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, +}; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{ArrowError, DataType, Field, Fields, Schema, SchemaRef, TimeUnit}; +use byteorder::{BigEndian, LittleEndian, ReadBytesExt}; + +fn timestamp_field() -> Field { + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + true, + ) +} + +fn sample_idx_field() -> Field { + Field::new("sample_idx", DataType::UInt16, true) +} + +fn tdf_field_read_string_to_string( + cursor: &mut Cursor<&[u8]>, + cursor_start: u64, + num: u8, + size: u8, +) -> Result { + let buf = crate::decoders::tdf_field_read_string(cursor, cursor_start, num, size)?; + + match String::from_utf8(buf) { + Ok(val) => Ok(val.trim_matches(char::from(0)).to_string()), + Err(..) => Ok(String::new()), + } +} + +fn finish_tdf_read(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result<()> { + let cursor_end = cursor.position(); + let cursor_read = cursor_end - cursor_start; + + if (size as u64) < cursor_read { + return Err(Error::new( + ErrorKind::InvalidData, + "Read overflow, corrupt data/metadata", + )); + } + + let underflow = size as u64 - cursor_read; + if underflow > 0 { + let mut buf = vec![0; underflow as usize]; + cursor.read_exact(&mut buf)?; + } + + Ok(()) +} + +pub fn tdf_parquet_schemas() -> Vec<(u16, &'static str, SchemaRef)> { + vec![ +{% for tdf_id, info in definitions.items() %} + ( + {{ tdf_id }}, + "{{ info['name'] }}", + tdf_parquet_schema({{ tdf_id }}).unwrap(), + ), +{% endfor %} + ] +} + +pub fn tdf_parquet_has_schema(tdf_id: u16) -> bool { + match tdf_id { +{% for tdf_id, info in definitions.items() %} + {{ tdf_id }} => true, +{% endfor %} + _ => false, + } +} + +pub fn tdf_parquet_schema(tdf_id: u16) -> Option { + match tdf_id { +{% for tdf_id, info in definitions.items() %} + {{ tdf_id }} => Some(Arc::new({{ info['arrow_schema'] }})), +{% endfor %} + _ => None, + } +} + +pub fn tdf_parquet_builder(tdf_id: u16, capacity: usize) -> Option { + match tdf_id { +{% for tdf_id, info in definitions.items() %} + {{ tdf_id }} => Some(TdfParquetBatchBuilder::{{ info['rust_variant_name'] }}( + {{ info['rust_builder_name'] }}::new(capacity), + )), +{% endfor %} + _ => None, + } +} + +#[derive(Clone, Copy, Debug)] +pub struct TdfParquetRowMeta { + pub time_unix_micros: Option, + pub sample_idx: Option, +} + +pub enum TdfParquetBatchBuilder { +{% for _tdf_id, info in definitions.items() %} + {{ info['rust_variant_name'] }}({{ info['rust_builder_name'] }}), +{% endfor %} +} + +impl TdfParquetBatchBuilder { + pub fn schema(&self) -> SchemaRef { + match self { +{% for _tdf_id, info in definitions.items() %} + Self::{{ info['rust_variant_name'] }}(builder) => builder.schema(), +{% endfor %} + } + } + + pub fn rows(&self) -> usize { + match self { +{% for _tdf_id, info in definitions.items() %} + Self::{{ info['rust_variant_name'] }}(builder) => builder.rows(), +{% endfor %} + } + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + match self { +{% for _tdf_id, info in definitions.items() %} + Self::{{ info['rust_variant_name'] }}(builder) => builder.append(meta, size, cursor), +{% endfor %} + } + } + + pub fn finish_batch(&mut self) -> std::result::Result { + match self { +{% for _tdf_id, info in definitions.items() %} + Self::{{ info['rust_variant_name'] }}(builder) => builder.finish_batch(), +{% endfor %} + } + } +} + +{% for tdf_id, info in definitions.items() %} +pub struct {{ info['rust_builder_name'] }} { +{% for name, ty in info['parquet_storage_fields'] %} + {{ name }}: {{ ty }}, +{% endfor %} +} + +impl {{ info['rust_builder_name'] }} { + pub fn new(capacity: usize) -> Self { + Self { +{% for init in info['parquet_init_fields'] %} + {{ init }}, +{% endfor %} + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema({{ tdf_id }}).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + +{% if info['parquet_has_lists'] %} + fn list_value_field(&self, field_index: usize) -> Arc { + let schema = self.schema(); + match schema.field(field_index).data_type() { + DataType::List(field) => field.clone(), + _ => unreachable!("generated list field index is not a list"), + } + } +{% endif %} + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + +{% for line in info['parquet_append_lines'] %} + {{ line }} +{% endfor %} + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ +{% for array in info['parquet_finish_arrays'] %} + {{ array }}, +{% endfor %} + ]; + + RecordBatch::try_new(schema, columns) + } +} + +{% endfor %} diff --git a/src/args.rs b/src/args.rs index be779ae..e6d5166 100644 --- a/src/args.rs +++ b/src/args.rs @@ -18,3 +18,20 @@ impl fmt::Display for BlockSizeOptions { } } } + +#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)] +pub enum OutputFormat { + #[value(name = "csv")] + CSV, + #[value(name = "parquet")] + PARQUET, +} + +impl fmt::Display for OutputFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OutputFormat::CSV => write!(f, "csv"), + OutputFormat::PARQUET => write!(f, "Parquet"), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 6bdeb07..cd26dd3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,7 @@ -use chrono::SecondsFormat; -use itertools::Itertools; use memmap::Mmap; use std::collections::HashMap; -use std::collections::hash_map::Entry; use std::fs::File; -use std::io::{self, BufRead, BufReader, BufWriter, Cursor, Write}; +use std::io; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::thread; @@ -13,6 +10,11 @@ use tdf::TdfOutput; pub mod args; pub mod fs_util; +mod output_common; +mod output_csv; +mod output_parquet; + +pub const DEFAULT_MAX_READINGS_PER_OUTPUT_FILE: usize = 0; pub trait ProgressReporter { /// Called when progress starts. Could be used to initialize the state or display a start message. @@ -25,123 +27,6 @@ pub trait ProgressReporter { fn stop(&mut self); } -pub struct TdfCsvWriter { - decoder_idx: usize, - output_folder: std::path::PathBuf, - output_unix: bool, - pub outputs: - HashMap<(Option, u16), (std::path::PathBuf, std::io::BufWriter)>, - output_cnt: HashMap<(Option, u16), usize>, -} -impl TdfCsvWriter { - pub fn new( - decoder_idx: usize, - output_folder: std::path::PathBuf, - output_unix_time: bool, - ) -> Self { - Self { - decoder_idx: decoder_idx, - output_folder: output_folder, - output_unix: output_unix_time, - outputs: HashMap::new(), - output_cnt: HashMap::new(), - } - } - - pub fn output_path(self: &Self, remote_id: Option, tdf_id: u16) -> Option { - self.outputs - .get(&(remote_id, tdf_id)) - .map(|(pathbuf, _)| pathbuf.clone()) - } -} - -impl TdfOutput for TdfCsvWriter { - fn write( - &mut self, - remote_id: Option, - tdf_id: u16, - tdf_time: i64, - tdf_idx: Option, - size: u8, - cursor: &mut Cursor<&[u8]>, - ) -> std::io::Result<()> { - // Create writer if it doesn't exist - let (_, writer) = match self.outputs.entry((remote_id, tdf_id)) { - Entry::Occupied(o) => o.into_mut(), - Entry::Vacant(v) => { - let id_prefix = match remote_id { - Some(id) => format!("_{:016x}", id), - None => "".to_string(), - }; - let fname = format!( - "{}_{}_{:05}.csv", - id_prefix, - tdf::decoders::tdf_name(&tdf_id), - self.decoder_idx - ); - let path = self.output_folder.join(fname); - let mut writer = std::io::BufWriter::new(std::fs::File::create(path.clone())?); - - // Write header into file - let heading = tdf::decoders_csv::tdf_fields(&tdf_id).join(","); - writer.write_all(format!("time,{}\n", heading).as_bytes())?; - - // Touch the count variable in case the decoding fails - *self - .output_cnt - .entry((remote_id, tdf_id.to_owned())) - .or_default() += 0; - - // Insert into hashmap and return - v.insert((path, writer)) - } - }; - - // Construct CSV line - let reading = tdf::decoders_csv::tdf_read_into_str(&tdf_id, size, cursor)?; - let time = match tdf_idx { - Some(idx) => { - // Use the index directly if provided - format!("{idx}") - } - None => match self.output_unix { - // Otherwise, format the time to a string - true => { - let (unix_seconds, unix_nano) = tdf::time::tdf_time_to_unix(tdf_time); - format!("{}.{:06}", unix_seconds, unix_nano / 1000) - } - false => { - let datetime = tdf::time::tdf_time_to_datetime(tdf_time).expect("Invalid time"); - datetime.to_rfc3339_opts(SecondsFormat::Micros, true) - } - }, - }; - - let line: String = format!("{},{}\n", time, reading); - - // Write line to output - writer.write_all(line.as_bytes())?; - - // Increment output counter - *self - .output_cnt - .entry((remote_id, tdf_id.to_owned())) - .or_default() += 1; - Ok(()) - } - - fn iter_written(&self) -> impl Iterator, u16), &usize)> { - self.output_cnt.iter() - } - - fn written(&self, remote_id: Option, tdf_id: u16) -> usize { - match self.output_cnt.get(&(remote_id, tdf_id)) { - Some(val) => *val, - None => 0, - } - } -} - pub fn merge_input_files( output_prefix: &String, input_files: &Vec, @@ -167,10 +52,12 @@ pub struct DecodeWorkerArgs { pub decoder_idx: usize, pub input_file: std::path::PathBuf, pub output_folder: std::path::PathBuf, + pub output_prefix: String, pub output_unix_time: bool, pub start_block: usize, pub num_blocks: usize, pub block_size: usize, + pub output_format: args::OutputFormat, } #[derive(Clone)] @@ -186,14 +73,11 @@ pub struct DecodeWorkerArgsReporter { pub reporter: T, } -pub fn worker_run_decode(mut args: DecodeWorkerArgsReporter) { +pub fn worker_run_decode( + mut args: DecodeWorkerArgsReporter, + mut writer: U, +) { let mut block_counter: HashMap = HashMap::new(); - let mut csv_writer = TdfCsvWriter::new( - args.decode_args.decoder_idx, - args.decode_args.output_folder, - args.decode_args.output_unix_time, - ); - // Open file let file = File::open(args.decode_args.input_file.clone()).unwrap(); let mmap = unsafe { Mmap::map(&file).unwrap() }; @@ -209,7 +93,7 @@ pub fn worker_run_decode(mut args: DecodeWorkerArgsReporter .chunks_exact(args.decode_args.block_size) .enumerate() { - match blocks::decode_block(&mut csv_writer, block) { + match blocks::decode_block(&mut writer, block) { Ok(block_type) => *block_counter.entry(block_type).or_default() += 1, Err(_) => *block_counter.entry(blocks::BlockTypes::ERROR).or_default() += 1, } @@ -223,11 +107,11 @@ pub fn worker_run_decode(mut args: DecodeWorkerArgsReporter // Push TDF stats into the output hashmap let mut tdf_stats = args.tdf_stats.lock().unwrap(); - for ((remote_id, tdf_id), tdf_cnt) in csv_writer.iter_written() { + for ((remote_id, tdf_id), tdf_cnt) in writer.iter_written() { let res = tdf_stats .entry((*remote_id, *tdf_id)) .or_insert_with(|| HashMap::new()); - let path = csv_writer.output_path(*remote_id, *tdf_id).unwrap(); + let path = writer.output_path(*remote_id, *tdf_id).unwrap(); res.insert( args.decode_args.decoder_idx, @@ -253,6 +137,9 @@ pub struct RunArgs { pub output_folder: PathBuf, pub output_prefix: String, pub output_unix_time: bool, + pub output_format: args::OutputFormat, + pub merge_output_files: bool, + pub max_readings_per_output_file: usize, pub copy_reporter: T, pub decode_reporter: T, pub merge_reporter: T, @@ -308,10 +195,12 @@ pub fn run( decoder_idx: idx, input_file: merged_file.clone(), output_folder: args.output_folder.clone(), + output_prefix: args.output_prefix.clone(), output_unix_time: args.output_unix_time, start_block: idx * blocks_per_worker, num_blocks: num, block_size: args.block_size, + output_format: args.output_format, }, block_stats: stats_block.clone(), tdf_stats: stats_tdf.clone(), @@ -323,7 +212,25 @@ pub fn run( let mut workers = vec![]; for worker_arg in worker_args.into_iter() { workers.push(thread::spawn(move || { - worker_run_decode(worker_arg); + match worker_arg.decode_args.output_format { + args::OutputFormat::CSV => { + let writer = output_csv::TdfCsvWriter::new( + worker_arg.decode_args.decoder_idx, + worker_arg.decode_args.output_folder.clone(), + worker_arg.decode_args.output_prefix.clone(), + worker_arg.decode_args.output_unix_time, + ); + worker_run_decode(worker_arg, writer); + } + args::OutputFormat::PARQUET => { + let writer = output_parquet::TdfParquetWriter::new( + worker_arg.decode_args.decoder_idx, + worker_arg.decode_args.output_folder.clone(), + worker_arg.decode_args.output_prefix.clone(), + ); + worker_run_decode(worker_arg, writer); + } + }; })); } @@ -333,62 +240,33 @@ pub fn run( } args.decode_reporter.stop(); - // Merge output files - let results = stats_tdf.lock().unwrap(); - let num_files: usize = results.values().map(|inner| inner.len()).sum(); - - args.merge_reporter.start("Merging output files", num_files); - - for ((remote_id, tdf_id), worker_outputs) in results.iter() { - let id_prefix = match remote_id { - Some(id) => format!("_{:016x}", id), - None => "".to_string(), - }; - let output_path = args.output_folder.join(format!( - "{}{}_{}.csv", - args.output_prefix, - id_prefix, - tdf::decoders::tdf_name(tdf_id) - )); - output_files.push(output_path.clone()); - let err_path = output_path.clone(); - let output_file = File::create(output_path).map_err(|e| { - io::Error::new( - e.kind(), - format!( - "Failed to create output file '{}': {}", - err_path.display(), - e - ), - ) - })?; - let mut output = BufWriter::new(output_file); - let mut write_headings = true; - - for worker in worker_outputs.keys().sorted() { - let input_path = worker_outputs[worker].output.clone(); - let input = BufReader::new(File::open(&input_path)?); - - // Copy from input to output - for (idx, line) in input.lines().flatten().enumerate() { - if idx == 0 && !write_headings { - continue; - } - output.write_all(line.as_bytes())?; - output.write_all(b"\n")?; + if args.merge_output_files { + match args.output_format { + args::OutputFormat::CSV => { + output_csv::merge(args, &mut output_files, &stats_tdf)?; + } + args::OutputFormat::PARQUET => { + output_parquet::merge_with_threshold( + args, + &mut output_files, + &stats_tdf, + args.max_readings_per_output_file, + )?; } - // Remove input file - std::fs::remove_file(input_path)?; - write_headings = false; - - args.merge_reporter.increment(1); } - output.flush()?; + } else { + let results = stats_tdf.lock().unwrap(); + let mut worker_output_files: Vec = results + .values() + .flat_map(|worker_outputs| worker_outputs.values().map(|output| output.output.clone())) + .collect(); + worker_output_files.sort(); + output_files.extend(worker_output_files); } - args.merge_reporter.stop(); let block = stats_block.lock().unwrap().clone(); let mut tdf = HashMap::new(); + let results = stats_tdf.lock().unwrap(); for ((remote_id, tdf_id), worker_outputs) in results.iter() { let values = tdf.entry(*remote_id).or_insert_with(|| HashMap::new()); diff --git a/src/main_cli.rs b/src/main_cli.rs index c9a7620..3a7ec30 100644 --- a/src/main_cli.rs +++ b/src/main_cli.rs @@ -1,5 +1,6 @@ use clap::Parser; use indicatif::{ProgressBar, ProgressStyle}; +use infuse_decoder::args; use std::collections::HashMap; use std::io; use std::path::PathBuf; @@ -56,11 +57,19 @@ struct Cli { /// Write Unix timestamps instead of UTC strings #[arg(short, long)] unix: bool, + #[arg(long, default_value_t = args::OutputFormat::CSV)] + format: args::OutputFormat, /// Verbose CLI output #[arg(short, long)] verbose: bool, #[arg(long, default_value_t = infuse_decoder::args::BlockSizeOptions::B512)] block_size: infuse_decoder::args::BlockSizeOptions, + /// Maximum readings per output file (0 is no limit) + #[arg(long, default_value_t = infuse_decoder::DEFAULT_MAX_READINGS_PER_OUTPUT_FILE)] + max_readings_per_output_file: usize, + /// Keep decoder worker output files instead of merging them into linearized outputs + #[arg(long = "no-linearize-output", alias = "no-merge-output-files")] + no_linearize_output: bool, } fn main() -> io::Result<()> { @@ -108,6 +117,9 @@ fn main() -> io::Result<()> { output_folder: args.output.clone(), output_prefix: output_prefix, output_unix_time: args.unix, + output_format: args.format, + merge_output_files: !args.no_linearize_output, + max_readings_per_output_file: args.max_readings_per_output_file, copy_reporter: IndicatifProgress::new(), decode_reporter: IndicatifProgress::new(), merge_reporter: IndicatifProgress::new(), diff --git a/src/main_gui.rs b/src/main_gui.rs index 13b2239..b842f8f 100644 --- a/src/main_gui.rs +++ b/src/main_gui.rs @@ -8,6 +8,7 @@ use std::{collections::HashMap, path::PathBuf}; use eframe::egui::{self, IconData}; use egui_extras::{Column, TableBuilder}; use image::GenericImageView; +use infuse_decoder::args::OutputFormat; use rfd::FileDialog; use infuse_decoder::args::BlockSizeOptions; @@ -78,8 +79,11 @@ impl infuse_decoder::ProgressReporter for SliderState { struct MyApp { time_mode: TimeOutput, + output_format: OutputFormat, + linearize_output_files: bool, device_id: u64, block_size: BlockSizeOptions, + max_readings_per_output_file: usize, error_msg: Option, input_path: Option, input_files: Option>>, @@ -124,8 +128,11 @@ impl Default for MyApp { Self { time_mode: TimeOutput::UTC, + output_format: OutputFormat::CSV, + linearize_output_files: true, device_id: 0, block_size: BlockSizeOptions::B512, + max_readings_per_output_file: infuse_decoder::DEFAULT_MAX_READINGS_PER_OUTPUT_FILE, error_msg: None, input_path: None, input_files: None, @@ -276,29 +283,57 @@ fn core_options(app: &mut MyApp, ui: &mut egui::Ui) { ui.label("Output Prefix"); ui.text_edit_singleline(&mut app.output_prefix); - ui.label(format!("(e.g. {}_BATTERY_STATE.csv)", app.output_prefix)); + let extension = match app.output_format { + OutputFormat::CSV => "csv", + OutputFormat::PARQUET => "parquet", + }; + ui.label(format!( + "(e.g. {}_BATTERY_STATE.{extension})", + app.output_prefix + )); ui.end_row(); }); } fn decode_options(app: &mut MyApp, ui: &mut egui::Ui) { ui.horizontal(|ui| { + ui.vertical(|ui| { + ui.label("Output Format"); + ui.radio_value(&mut app.output_format, OutputFormat::CSV, "CSV"); + ui.radio_value(&mut app.output_format, OutputFormat::PARQUET, "Parquet"); + }); + ui.separator(); + ui.vertical(|ui| { + ui.label("File Output Control"); + ui.checkbox(&mut app.linearize_output_files, "Linearize Output"); + ui.label("Max Readings Per File"); + ui.add_enabled_ui(app.linearize_output_files, |ui| { + ui.add( + egui::DragValue::new(&mut app.max_readings_per_output_file) + .range(0..=usize::MAX) + .speed(10_000), + ); + }); + }); + ui.separator(); ui.vertical(|ui| { ui.label("Time Output Format"); - ui.radio_value( - &mut app.time_mode, - TimeOutput::UTC, - "UTC (2020-01-01T00:00:00.000000Z)", - ); - ui.radio_value( - &mut app.time_mode, - TimeOutput::UNIX, - "UNIX (1577800800.000000)", - ); + ui.add_enabled_ui(app.output_format == OutputFormat::CSV, |ui| { + ui.radio_value( + &mut app.time_mode, + TimeOutput::UTC, + "UTC (2020-01-01T00:00:00.000000Z)", + ); + ui.radio_value( + &mut app.time_mode, + TimeOutput::UNIX, + "UNIX (1577800800.000000)", + ); + }); }); ui.separator(); ui.vertical(|ui| { - ui.label("Block Size"); + ui.label("Input Block Size"); egui::ComboBox::from_id_salt("Block Size") .selected_text(format!("{:}", app.block_size)) .show_ui(ui, |ui| { @@ -347,6 +382,9 @@ fn start_button(app: &mut MyApp, ui: &mut egui::Ui) { output_folder: app.output_folder.clone(), output_prefix: app.output_prefix.clone(), output_unix_time: app.time_mode == TimeOutput::UNIX, + output_format: app.output_format, + merge_output_files: app.linearize_output_files, + max_readings_per_output_file: app.max_readings_per_output_file, copy_reporter: app.progress_copy.clone(), decode_reporter: app.progress_decode.clone(), merge_reporter: app.progress_merge.clone(), diff --git a/src/output_common.rs b/src/output_common.rs new file mode 100644 index 0000000..726aaf6 --- /dev/null +++ b/src/output_common.rs @@ -0,0 +1,95 @@ +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +pub(crate) type OutputKey = (Option, u16); + +pub(crate) fn worker_output_path( + output_folder: &Path, + output_prefix: &str, + remote_id: Option, + tdf_id: u16, + decoder_idx: usize, + extension: &str, +) -> PathBuf { + let mut fname_parts = Vec::new(); + if !output_prefix.is_empty() { + fname_parts.push(output_prefix.to_string()); + } + if let Some(id) = remote_id { + fname_parts.push(format!("{id:016x}")); + } + fname_parts.push(tdf::decoders::tdf_name(&tdf_id).to_string()); + + output_folder.join(format!( + "{}_{:05}.{}", + fname_parts.join("_"), + decoder_idx, + extension + )) +} + +pub(crate) fn merged_output_path( + output_folder: &Path, + output_prefix: &str, + remote_id: Option, + tdf_id: u16, + part_idx: Option, + extension: &str, +) -> PathBuf { + let id_prefix = match remote_id { + Some(id) => format!("_{id:016x}"), + None => String::new(), + }; + let name = tdf::decoders::tdf_name(&tdf_id); + + match part_idx { + Some(part_idx) => output_folder.join(format!( + "{}{}_{}_{:05}.{}", + output_prefix, id_prefix, name, part_idx, extension + )), + None => output_folder.join(format!( + "{}{}_{}.{}", + output_prefix, id_prefix, name, extension + )), + } +} + +pub(crate) fn rename_first_file_if_splitting( + part_idx: usize, + output_files: &mut [PathBuf], + plain_path: PathBuf, + numbered_path: PathBuf, +) -> io::Result<()> { + if part_idx != 1 || plain_path == numbered_path || !plain_path.exists() { + return Ok(()); + } + + match fs::rename(&plain_path, &numbered_path) { + Ok(()) => {} + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { + fs::remove_file(&numbered_path)?; + fs::rename(&plain_path, &numbered_path)?; + } + Err(err) => return Err(err), + } + + if let Some(path) = output_files.first_mut() { + *path = numbered_path; + } + + Ok(()) +} + +pub(crate) fn touch_output_count(output_cnt: &mut HashMap, key: OutputKey) { + output_cnt.entry(key).or_default(); +} + +pub(crate) fn increment_output_count(output_cnt: &mut HashMap, key: OutputKey) { + *output_cnt.entry(key).or_default() += 1; +} + +pub(crate) fn written(output_cnt: &HashMap, key: OutputKey) -> usize { + output_cnt.get(&key).copied().unwrap_or_default() +} diff --git a/src/output_csv.rs b/src/output_csv.rs new file mode 100644 index 0000000..10924ec --- /dev/null +++ b/src/output_csv.rs @@ -0,0 +1,365 @@ +use chrono::SecondsFormat; +use itertools::Itertools; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::{self, File}; +use std::io::Cursor; +use std::io::{self, BufRead, BufReader, BufWriter, Write}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use tdf::TdfOutput; + +use crate::output_common::{ + OutputKey, increment_output_count, merged_output_path, rename_first_file_if_splitting, + touch_output_count, worker_output_path, written, +}; +use crate::{ProgressReporter, RunArgs, TdfDecoderOutputs}; + +pub struct TdfCsvWriter { + decoder_idx: usize, + output_folder: std::path::PathBuf, + output_prefix: String, + output_unix: bool, + pub outputs: + HashMap<(Option, u16), (std::path::PathBuf, std::io::BufWriter)>, + output_cnt: HashMap, +} +impl TdfCsvWriter { + pub fn new( + decoder_idx: usize, + output_folder: std::path::PathBuf, + output_prefix: String, + output_unix_time: bool, + ) -> Self { + Self { + decoder_idx: decoder_idx, + output_folder: output_folder, + output_prefix, + output_unix: output_unix_time, + outputs: HashMap::new(), + output_cnt: HashMap::new(), + } + } +} + +impl TdfOutput for TdfCsvWriter { + fn output_path(self: &Self, remote_id: Option, tdf_id: u16) -> Option { + self.outputs + .get(&(remote_id, tdf_id)) + .map(|(pathbuf, _)| pathbuf.clone()) + } + + fn write( + &mut self, + remote_id: Option, + tdf_id: u16, + tdf_time: i64, + tdf_idx: Option, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> std::io::Result<()> { + // Create writer if it doesn't exist + let (_, writer) = match self.outputs.entry((remote_id, tdf_id)) { + Entry::Occupied(o) => o.into_mut(), + Entry::Vacant(v) => { + let path = worker_output_path( + &self.output_folder, + &self.output_prefix, + remote_id, + tdf_id, + self.decoder_idx, + "csv", + ); + let mut writer = std::io::BufWriter::new(std::fs::File::create(path.clone())?); + + // Write header into file + let heading = tdf::decoders_csv::tdf_fields(&tdf_id).join(","); + writer.write_all(format!("time,{}\n", heading).as_bytes())?; + + // Touch the count variable in case the decoding fails + touch_output_count(&mut self.output_cnt, (remote_id, tdf_id)); + + // Insert into hashmap and return + v.insert((path, writer)) + } + }; + + // Construct CSV line + let reading = tdf::decoders_csv::tdf_read_into_str(&tdf_id, size, cursor)?; + let time = match tdf_idx { + Some(idx) => { + // Use the index directly if provided + format!("{idx}") + } + None => match self.output_unix { + // Otherwise, format the time to a string + true => { + let (unix_seconds, unix_nano) = tdf::time::tdf_time_to_unix(tdf_time); + format!("{}.{:06}", unix_seconds, unix_nano / 1000) + } + false => { + let datetime = tdf::time::tdf_time_to_datetime(tdf_time).expect("Invalid time"); + datetime.to_rfc3339_opts(SecondsFormat::Micros, true) + } + }, + }; + + let line: String = format!("{},{}\n", time, reading); + + // Write line to output + writer.write_all(line.as_bytes())?; + + // Increment output counter + increment_output_count(&mut self.output_cnt, (remote_id, tdf_id)); + Ok(()) + } + + fn iter_written(&self) -> impl Iterator, u16), &usize)> { + self.output_cnt.iter() + } + + fn written(&self, remote_id: Option, tdf_id: u16) -> usize { + written(&self.output_cnt, (remote_id, tdf_id)) + } +} + +struct TdfCsvMergedOutput { + output_folder: PathBuf, + output_prefix: String, + remote_id: Option, + tdf_id: u16, + max_readings_per_file: Option, + output_files: Vec, + writer: Option>, + header: Option, + readings_in_file: usize, + part_idx: usize, +} + +impl TdfCsvMergedOutput { + fn new( + output_folder: PathBuf, + output_prefix: String, + remote_id: Option, + tdf_id: u16, + max_readings_per_file: usize, + ) -> Self { + Self { + output_folder, + output_prefix, + remote_id, + tdf_id, + max_readings_per_file: match max_readings_per_file { + 0 => None, + value => Some(value), + }, + output_files: Vec::new(), + writer: None, + header: None, + readings_in_file: 0, + part_idx: 0, + } + } + + fn set_header(&mut self, header: String) -> io::Result<()> { + if self.header.is_none() { + self.header = Some(header); + self.start_next_file()?; + } + Ok(()) + } + + fn append_line(&mut self, line: &str) -> io::Result<()> { + if self.writer.is_none() { + self.start_next_file()?; + } + + if self + .max_readings_per_file + .is_some_and(|max_readings| self.readings_in_file >= max_readings) + { + self.start_next_file()?; + } + + self.writer + .as_mut() + .expect("CSV writer should be open") + .write_all(line.as_bytes())?; + self.writer + .as_mut() + .expect("CSV writer should be open") + .write_all(b"\n")?; + self.readings_in_file += 1; + Ok(()) + } + + fn finish(&mut self) -> io::Result> { + self.finish_current_file()?; + Ok(std::mem::take(&mut self.output_files)) + } + + fn start_next_file(&mut self) -> io::Result<()> { + self.finish_current_file()?; + self.rename_first_file_if_splitting()?; + + let path = self.output_path(); + let err_path = path.clone(); + let file = File::create(path.clone()).map_err(|e| { + io::Error::new( + e.kind(), + format!( + "Failed to create output file '{}': {}", + err_path.display(), + e + ), + ) + })?; + let mut writer = BufWriter::new(file); + + if let Some(header) = &self.header { + writer.write_all(header.as_bytes())?; + writer.write_all(b"\n")?; + self.readings_in_file = 0; + } else { + self.readings_in_file = 0; + } + + self.output_files.push(path); + self.writer = Some(writer); + self.part_idx += 1; + Ok(()) + } + + fn rename_first_file_if_splitting(&mut self) -> io::Result<()> { + let plain_path = self.plain_output_path(); + let numbered_path = self.numbered_output_path(0); + + rename_first_file_if_splitting( + self.part_idx, + &mut self.output_files, + plain_path, + numbered_path, + ) + } + + fn finish_current_file(&mut self) -> io::Result<()> { + if let Some(mut writer) = self.writer.take() { + writer.flush()?; + } + Ok(()) + } + + fn output_path(&self) -> PathBuf { + if self.part_idx == 0 { + self.plain_output_path() + } else { + self.numbered_output_path(self.part_idx) + } + } + + fn plain_output_path(&self) -> PathBuf { + merged_output_path( + &self.output_folder, + &self.output_prefix, + self.remote_id, + self.tdf_id, + None, + "csv", + ) + } + + fn numbered_output_path(&self, part_idx: usize) -> PathBuf { + merged_output_path( + &self.output_folder, + &self.output_prefix, + self.remote_id, + self.tdf_id, + Some(part_idx), + "csv", + ) + } +} + +pub fn merge( + args: &mut RunArgs, + output_files: &mut Vec, + stats_tdf: &Arc, u16), HashMap>>>, +) -> io::Result<()> { + let results = stats_tdf.lock().unwrap(); + let num_files: usize = results.values().map(|inner| inner.len()).sum(); + + args.merge_reporter.start("Merging output files", num_files); + + for ((remote_id, tdf_id), worker_outputs) in results.iter() { + let mut output = TdfCsvMergedOutput::new( + args.output_folder.clone(), + args.output_prefix.clone(), + *remote_id, + *tdf_id, + args.max_readings_per_output_file, + ); + + for worker in worker_outputs.keys().sorted() { + let input_path = worker_outputs[worker].output.clone(); + let input = BufReader::new(File::open(&input_path)?); + + for (idx, line) in input.lines().enumerate() { + let line = line?; + if idx == 0 { + output.set_header(line)?; + continue; + } + output.append_line(&line)?; + } + + fs::remove_file(input_path)?; + + args.merge_reporter.increment(1); + } + output_files.extend(output.finish()?); + } + args.merge_reporter.stop(); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + + fn unique_temp_dir(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("infuse_decoder_{name}_{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn zero_max_readings_keeps_csv_output_in_one_file() { + let output_dir = unique_temp_dir("zero_max_readings_csv"); + let mut output = TdfCsvMergedOutput::new(output_dir.clone(), "out".to_string(), None, 1, 0); + + output.set_header("time,value".to_string()).unwrap(); + output.append_line("1,10").unwrap(); + output.append_line("2,20").unwrap(); + output.append_line("3,30").unwrap(); + + let files = output.finish().unwrap(); + assert_eq!(files.len(), 1); + assert_eq!( + files[0], + output_dir.join(format!("out_{}.csv", tdf::decoders::tdf_name(&1))) + ); + + let mut contents = String::new(); + File::open(&files[0]) + .unwrap() + .read_to_string(&mut contents) + .unwrap(); + assert_eq!(contents, "time,value\n1,10\n2,20\n3,30\n"); + + fs::remove_dir_all(output_dir).unwrap(); + } +} diff --git a/src/output_parquet.rs b/src/output_parquet.rs new file mode 100644 index 0000000..75d0660 --- /dev/null +++ b/src/output_parquet.rs @@ -0,0 +1,415 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::fs::File; +use std::io::{self, Cursor, Read}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; +use itertools::Itertools; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use tdf::TdfOutput; +use tdf::decoders_parquet::{TdfParquetBatchBuilder, TdfParquetRowMeta}; + +use crate::output_common::{ + OutputKey, increment_output_count, merged_output_path, rename_first_file_if_splitting, + touch_output_count, worker_output_path, written, +}; +use crate::{ProgressReporter, RunArgs, TdfDecoderOutputs}; + +const DEFAULT_BATCH_ROWS: usize = 65536; + +struct TdfParquetOutputFile { + path: PathBuf, + tdf_id: u16, + builder: TdfParquetBatchBuilder, + writer: ArrowWriter, + finished: bool, +} + +impl TdfParquetOutputFile { + fn new(path: PathBuf, tdf_id: u16, batch_rows: usize) -> io::Result { + let builder = tdf::decoders_parquet::tdf_parquet_builder(tdf_id, batch_rows) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Unknown TDF ID"))?; + let file = File::create(path.clone())?; + let writer = ArrowWriter::try_new(file, builder.schema(), None).map_err(to_io_error)?; + + Ok(Self { + path, + tdf_id, + builder, + writer, + finished: false, + }) + } + + fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> io::Result<()> { + self.builder.append(meta, size, cursor)?; + Ok(()) + } + + fn flush_batch(&mut self, batch_rows: usize) -> io::Result<()> { + if self.builder.rows() == 0 { + return Ok(()); + } + + let batch = self.builder.finish_batch().map_err(to_io_error)?; + self.writer.write(&batch).map_err(to_io_error)?; + self.builder = tdf::decoders_parquet::tdf_parquet_builder(self.tdf_id, batch_rows) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Unknown TDF ID"))?; + Ok(()) + } + + fn finish(&mut self, batch_rows: usize) -> io::Result<()> { + if self.finished { + return Ok(()); + } + + self.flush_batch(batch_rows)?; + self.writer.finish().map_err(to_io_error)?; + self.finished = true; + Ok(()) + } +} + +impl Drop for TdfParquetOutputFile { + fn drop(&mut self) { + let _ = self.finish(DEFAULT_BATCH_ROWS); + } +} + +pub struct TdfParquetWriter { + decoder_idx: usize, + output_folder: PathBuf, + output_prefix: String, + batch_rows: usize, + outputs: HashMap<(Option, u16), TdfParquetOutputFile>, + output_cnt: HashMap, +} + +impl TdfParquetWriter { + pub fn new(decoder_idx: usize, output_folder: PathBuf, output_prefix: String) -> Self { + Self::new_with_batch_rows( + decoder_idx, + output_folder, + output_prefix, + DEFAULT_BATCH_ROWS, + ) + } + + pub fn new_with_batch_rows( + decoder_idx: usize, + output_folder: PathBuf, + output_prefix: String, + batch_rows: usize, + ) -> Self { + Self { + decoder_idx, + output_folder, + output_prefix, + batch_rows: batch_rows.max(1), + outputs: HashMap::new(), + output_cnt: HashMap::new(), + } + } + + pub fn finish(&mut self) -> io::Result<()> { + for output in self.outputs.values_mut() { + output.finish(self.batch_rows)?; + } + Ok(()) + } + + fn create_output( + decoder_idx: usize, + output_folder: &std::path::Path, + output_prefix: &str, + batch_rows: usize, + remote_id: Option, + tdf_id: u16, + ) -> io::Result { + let path = worker_output_path( + output_folder, + output_prefix, + remote_id, + tdf_id, + decoder_idx, + "parquet", + ); + + TdfParquetOutputFile::new(path, tdf_id, batch_rows) + } +} + +impl Drop for TdfParquetWriter { + fn drop(&mut self) { + let _ = self.finish(); + } +} + +impl TdfOutput for TdfParquetWriter { + fn output_path(&self, remote_id: Option, tdf_id: u16) -> Option { + self.outputs + .get(&(remote_id, tdf_id)) + .map(|output| output.path.clone()) + } + + fn write( + &mut self, + remote_id: Option, + tdf_id: u16, + tdf_time: i64, + tdf_idx: Option, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> io::Result<()> { + if !tdf::decoders_parquet::tdf_parquet_has_schema(tdf_id) { + let mut buf = vec![0; size as usize]; + cursor.read_exact(&mut buf)?; + return Ok(()); + } + + let output = match self.outputs.entry((remote_id, tdf_id)) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let output = Self::create_output( + self.decoder_idx, + &self.output_folder, + &self.output_prefix, + self.batch_rows, + remote_id, + tdf_id, + )?; + + touch_output_count(&mut self.output_cnt, (remote_id, tdf_id)); + entry.insert(output) + } + }; + + let meta = match tdf_idx { + Some(idx) => TdfParquetRowMeta { + time_unix_micros: None, + sample_idx: Some(idx), + }, + None => TdfParquetRowMeta { + time_unix_micros: Some(tdf::time::tdf_time_to_unix_micros(tdf_time)), + sample_idx: None, + }, + }; + + output.append(meta, size, cursor)?; + + if output.builder.rows() >= self.batch_rows { + output.flush_batch(self.batch_rows)?; + } + + increment_output_count(&mut self.output_cnt, (remote_id, tdf_id)); + + Ok(()) + } + + fn iter_written(&self) -> impl Iterator, u16), &usize)> { + self.output_cnt.iter() + } + + fn written(&self, remote_id: Option, tdf_id: u16) -> usize { + written(&self.output_cnt, (remote_id, tdf_id)) + } +} + +struct TdfParquetMergedOutput { + output_folder: PathBuf, + output_prefix: String, + remote_id: Option, + tdf_id: u16, + threshold_rows: Option, + output_files: Vec, + writer: Option>, + rows_in_file: usize, + part_idx: usize, +} + +impl TdfParquetMergedOutput { + fn new( + output_folder: PathBuf, + output_prefix: String, + remote_id: Option, + tdf_id: u16, + threshold_rows: usize, + ) -> Self { + Self { + output_folder, + output_prefix, + remote_id, + tdf_id, + threshold_rows: match threshold_rows { + 0 => None, + value => Some(value), + }, + output_files: Vec::new(), + writer: None, + rows_in_file: 0, + part_idx: 0, + } + } + + fn append_batch(&mut self, batch: &RecordBatch) -> io::Result<()> { + let mut offset = 0; + + while offset < batch.num_rows() { + if self.writer.is_none() + || self + .threshold_rows + .is_some_and(|threshold| self.rows_in_file >= threshold) + { + self.start_next_file(batch.schema())?; + } + + let rows_to_write = match self.threshold_rows { + Some(threshold) => { + let rows_remaining = threshold - self.rows_in_file; + rows_remaining.min(batch.num_rows() - offset) + } + None => batch.num_rows() - offset, + }; + let batch = batch.slice(offset, rows_to_write); + + self.writer + .as_mut() + .expect("Parquet writer should be open") + .write(&batch) + .map_err(to_io_error)?; + + self.rows_in_file += rows_to_write; + offset += rows_to_write; + } + + Ok(()) + } + + fn finish(&mut self) -> io::Result> { + self.finish_current_file()?; + Ok(std::mem::take(&mut self.output_files)) + } + + fn start_next_file(&mut self, schema: SchemaRef) -> io::Result<()> { + self.finish_current_file()?; + self.rename_first_file_if_splitting()?; + + let path = self.output_path(); + let file = File::create(path.clone())?; + let writer = ArrowWriter::try_new(file, schema, None).map_err(to_io_error)?; + + self.output_files.push(path); + self.writer = Some(writer); + self.rows_in_file = 0; + self.part_idx += 1; + + Ok(()) + } + + fn rename_first_file_if_splitting(&mut self) -> io::Result<()> { + let plain_path = self.plain_output_path(); + let numbered_path = self.numbered_output_path(0); + + rename_first_file_if_splitting( + self.part_idx, + &mut self.output_files, + plain_path, + numbered_path, + ) + } + + fn finish_current_file(&mut self) -> io::Result<()> { + if let Some(mut writer) = self.writer.take() { + writer.finish().map_err(to_io_error)?; + } + Ok(()) + } + + fn output_path(&self) -> PathBuf { + if self.part_idx == 0 { + self.plain_output_path() + } else { + self.numbered_output_path(self.part_idx) + } + } + + fn plain_output_path(&self) -> PathBuf { + merged_output_path( + &self.output_folder, + &self.output_prefix, + self.remote_id, + self.tdf_id, + None, + "parquet", + ) + } + + fn numbered_output_path(&self, part_idx: usize) -> PathBuf { + merged_output_path( + &self.output_folder, + &self.output_prefix, + self.remote_id, + self.tdf_id, + Some(part_idx), + "parquet", + ) + } +} + +pub fn merge_with_threshold( + args: &mut RunArgs, + output_files: &mut Vec, + stats_tdf: &Arc, u16), HashMap>>>, + threshold_rows: usize, +) -> io::Result<()> { + let results = stats_tdf.lock().unwrap(); + let num_files: usize = results.values().map(|inner| inner.len()).sum(); + + args.merge_reporter.start("Merging output files", num_files); + + for ((remote_id, tdf_id), worker_outputs) in results.iter() { + let mut output = TdfParquetMergedOutput::new( + args.output_folder.clone(), + args.output_prefix.clone(), + *remote_id, + *tdf_id, + threshold_rows, + ); + + for worker in worker_outputs.keys().sorted() { + let input_path = worker_outputs[worker].output.clone(); + let file = File::open(&input_path)?; + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .map_err(to_io_error)? + .with_batch_size(DEFAULT_BATCH_ROWS) + .build() + .map_err(to_io_error)?; + + for batch in reader { + output.append_batch(&batch.map_err(to_io_error)?)?; + } + + std::fs::remove_file(input_path)?; + args.merge_reporter.increment(1); + } + + output_files.extend(output.finish()?); + } + + args.merge_reporter.stop(); + + Ok(()) +} + +fn to_io_error(err: E) -> io::Error { + io::Error::new(io::ErrorKind::Other, err) +} diff --git a/tdf/Cargo.toml b/tdf/Cargo.toml index 72b9d0a..0b9ccc8 100644 --- a/tdf/Cargo.toml +++ b/tdf/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition = "2021" [dependencies] +arrow-array = "59.0.0" +arrow-buffer = "59.0.0" +arrow-schema = "59.0.0" byteorder = "1.5.0" chrono = "0.4.38" hex = "0.4.3" diff --git a/tdf/src/decoders.rs b/tdf/src/decoders.rs index e6cad67..7c00032 100644 --- a/tdf/src/decoders.rs +++ b/tdf/src/decoders.rs @@ -1,6 +1,7 @@ -use std::io::{Cursor, Error, ErrorKind, Read, Result}; +use std::io::{Cursor, Read, Result, Error, ErrorKind}; -pub fn tdf_name(tdf_id: &u16) -> String { +pub fn tdf_name(tdf_id: &u16) -> String +{ match tdf_id { 1 => String::from("ANNOUNCE"), 2 => String::from("BATTERY_STATE"), @@ -66,11 +67,8 @@ pub fn tdf_name(tdf_id: &u16) -> String { } } -pub fn vla_bytes_remaining( - cursor: &mut Cursor<&[u8]>, - cursor_start: u64, - size: u8, -) -> Result { +pub fn vla_bytes_remaining(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result +{ let cursor_current = cursor.position(); let cursor_read = cursor_current - cursor_start; if cursor_read >= size as u64 { @@ -84,12 +82,8 @@ pub fn vla_bytes_remaining( Ok(bytes_remaining as usize) } -pub fn tdf_field_read_string( - cursor: &mut Cursor<&[u8]>, - cursor_start: u64, - num: u8, - size: u8, -) -> Result> { +pub fn tdf_field_read_string(cursor: &mut Cursor<&[u8]>, cursor_start: u64, num: u8, size: u8) -> Result> +{ let string_length = match num { 0 => vla_bytes_remaining(cursor, cursor_start, size)?, _ => num as usize, @@ -101,11 +95,8 @@ pub fn tdf_field_read_string( Ok(buf) } -pub fn tdf_field_read_vla( - cursor: &mut Cursor<&[u8]>, - cursor_start: u64, - size: u8, -) -> Result> { +pub fn tdf_field_read_vla(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result> +{ let bytes_remaining = vla_bytes_remaining(cursor, cursor_start, size)?; let mut buf = vec![0u8; bytes_remaining]; diff --git a/tdf/src/decoders_parquet.rs b/tdf/src/decoders_parquet.rs new file mode 100644 index 0000000..ecbf66f --- /dev/null +++ b/tdf/src/decoders_parquet.rs @@ -0,0 +1,5593 @@ +use std::io::{Cursor, Error, ErrorKind, Read, Result}; +use std::sync::Arc; + +use arrow_array::{ + ArrayRef, BinaryArray, FixedSizeListArray, Float32Array, Float64Array, Int16Array, Int32Array, + Int8Array, ListArray, RecordBatch, StringArray, StructArray, TimestampMicrosecondArray, + UInt16Array, UInt32Array, UInt64Array, UInt8Array, +}; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{ArrowError, DataType, Field, Fields, Schema, SchemaRef, TimeUnit}; +use byteorder::{BigEndian, LittleEndian, ReadBytesExt}; + +fn timestamp_field() -> Field { + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + true, + ) +} + +fn sample_idx_field() -> Field { + Field::new("sample_idx", DataType::UInt16, true) +} + +fn tdf_field_read_string_to_string( + cursor: &mut Cursor<&[u8]>, + cursor_start: u64, + num: u8, + size: u8, +) -> Result { + let buf = crate::decoders::tdf_field_read_string(cursor, cursor_start, num, size)?; + + match String::from_utf8(buf) { + Ok(val) => Ok(val.trim_matches(char::from(0)).to_string()), + Err(..) => Ok(String::new()), + } +} + +fn finish_tdf_read(cursor: &mut Cursor<&[u8]>, cursor_start: u64, size: u8) -> Result<()> { + let cursor_end = cursor.position(); + let cursor_read = cursor_end - cursor_start; + + if (size as u64) < cursor_read { + return Err(Error::new( + ErrorKind::InvalidData, + "Read overflow, corrupt data/metadata", + )); + } + + let underflow = size as u64 - cursor_read; + if underflow > 0 { + let mut buf = vec![0; underflow as usize]; + cursor.read_exact(&mut buf)?; + } + + Ok(()) +} + +pub fn tdf_parquet_schemas() -> Vec<(u16, &'static str, SchemaRef)> { + vec![ + (1, "ANNOUNCE", tdf_parquet_schema(1).unwrap()), + (2, "BATTERY_STATE", tdf_parquet_schema(2).unwrap()), + (3, "AMBIENT_TEMP_PRES_HUM", tdf_parquet_schema(3).unwrap()), + (4, "AMBIENT_TEMPERATURE", tdf_parquet_schema(4).unwrap()), + (5, "TIME_SYNC", tdf_parquet_schema(5).unwrap()), + (6, "REBOOT_INFO", tdf_parquet_schema(6).unwrap()), + (7, "ANNOUNCE_V2", tdf_parquet_schema(7).unwrap()), + (8, "SOC_TEMPERATURE", tdf_parquet_schema(8).unwrap()), + (10, "ACC_2G", tdf_parquet_schema(10).unwrap()), + (11, "ACC_4G", tdf_parquet_schema(11).unwrap()), + (12, "ACC_8G", tdf_parquet_schema(12).unwrap()), + (13, "ACC_16G", tdf_parquet_schema(13).unwrap()), + (14, "GYR_125DPS", tdf_parquet_schema(14).unwrap()), + (15, "GYR_250DPS", tdf_parquet_schema(15).unwrap()), + (16, "GYR_500DPS", tdf_parquet_schema(16).unwrap()), + (17, "GYR_1000DPS", tdf_parquet_schema(17).unwrap()), + (18, "GYR_2000DPS", tdf_parquet_schema(18).unwrap()), + (19, "GCS_WGS84_LLHA", tdf_parquet_schema(19).unwrap()), + (20, "UBX_NAV_PVT", tdf_parquet_schema(20).unwrap()), + (21, "LTE_CONN_STATUS", tdf_parquet_schema(21).unwrap()), + (22, "GLOBALSTAR_PKT", tdf_parquet_schema(22).unwrap()), + (23, "ACC_MAGNITUDE_STD_DEV", tdf_parquet_schema(23).unwrap()), + (24, "ACTIVITY_METRIC", tdf_parquet_schema(24).unwrap()), + (25, "ALGORITHM_OUTPUT", tdf_parquet_schema(25).unwrap()), + (26, "RUNTIME_ERROR", tdf_parquet_schema(26).unwrap()), + (27, "CHARGER_EN_CONTROL", tdf_parquet_schema(27).unwrap()), + (28, "GNSS_FIX_INFO", tdf_parquet_schema(28).unwrap()), + (29, "BLUETOOTH_CONNECTION", tdf_parquet_schema(29).unwrap()), + (30, "BLUETOOTH_RSSI", tdf_parquet_schema(30).unwrap()), + ( + 31, + "BLUETOOTH_DATA_THROUGHPUT", + tdf_parquet_schema(31).unwrap(), + ), + ( + 32, + "ALGORITHM_CLASS_HISTOGRAM", + tdf_parquet_schema(32).unwrap(), + ), + ( + 33, + "ALGORITHM_CLASS_TIME_SERIES", + tdf_parquet_schema(33).unwrap(), + ), + (34, "LTE_TAC_CELLS", tdf_parquet_schema(34).unwrap()), + (35, "WIFI_AP_INFO", tdf_parquet_schema(35).unwrap()), + (36, "DEVICE_TILT", tdf_parquet_schema(36).unwrap()), + (37, "NRF9X_GNSS_PVT", tdf_parquet_schema(37).unwrap()), + ( + 38, + "BATTERY_CHARGE_ACCUMULATED", + tdf_parquet_schema(38).unwrap(), + ), + (39, "INFUSE_BLUETOOTH_RSSI", tdf_parquet_schema(39).unwrap()), + (40, "ADC_RAW_8", tdf_parquet_schema(40).unwrap()), + (41, "ADC_RAW_16", tdf_parquet_schema(41).unwrap()), + (42, "ADC_RAW_32", tdf_parquet_schema(42).unwrap()), + (43, "ANNOTATION", tdf_parquet_schema(43).unwrap()), + (44, "LORA_RX", tdf_parquet_schema(44).unwrap()), + (45, "LORA_TX", tdf_parquet_schema(45).unwrap()), + (46, "IDX_ARRAY_FREQ", tdf_parquet_schema(46).unwrap()), + (47, "IDX_ARRAY_PERIOD", tdf_parquet_schema(47).unwrap()), + (48, "WIFI_CONNECTED", tdf_parquet_schema(48).unwrap()), + ( + 49, + "WIFI_CONNECTION_FAILED", + tdf_parquet_schema(49).unwrap(), + ), + (50, "WIFI_DISCONNECTED", tdf_parquet_schema(50).unwrap()), + (51, "NETWORK_SCAN_COUNT", tdf_parquet_schema(51).unwrap()), + (52, "EXCEPTION_STACK_FRAME", tdf_parquet_schema(52).unwrap()), + (53, "BATTERY_VOLTAGE", tdf_parquet_schema(53).unwrap()), + (54, "BATTERY_SOC", tdf_parquet_schema(54).unwrap()), + (55, "STATE_EVENT_SET", tdf_parquet_schema(55).unwrap()), + (56, "STATE_EVENT_CLEARED", tdf_parquet_schema(56).unwrap()), + (57, "STATE_DURATION", tdf_parquet_schema(57).unwrap()), + (58, "PCM_16BIT_CHAN_LEFT", tdf_parquet_schema(58).unwrap()), + (59, "PCM_16BIT_CHAN_RIGHT", tdf_parquet_schema(59).unwrap()), + (60, "PCM_16BIT_CHAN_DUAL", tdf_parquet_schema(60).unwrap()), + (61, "KVS_VALUE_CHANGED", tdf_parquet_schema(61).unwrap()), + ] +} + +pub fn tdf_parquet_has_schema(tdf_id: u16) -> bool { + match tdf_id { + 1 => true, + 2 => true, + 3 => true, + 4 => true, + 5 => true, + 6 => true, + 7 => true, + 8 => true, + 10 => true, + 11 => true, + 12 => true, + 13 => true, + 14 => true, + 15 => true, + 16 => true, + 17 => true, + 18 => true, + 19 => true, + 20 => true, + 21 => true, + 22 => true, + 23 => true, + 24 => true, + 25 => true, + 26 => true, + 27 => true, + 28 => true, + 29 => true, + 30 => true, + 31 => true, + 32 => true, + 33 => true, + 34 => true, + 35 => true, + 36 => true, + 37 => true, + 38 => true, + 39 => true, + 40 => true, + 41 => true, + 42 => true, + 43 => true, + 44 => true, + 45 => true, + 46 => true, + 47 => true, + 48 => true, + 49 => true, + 50 => true, + 51 => true, + 52 => true, + 53 => true, + 54 => true, + 55 => true, + 56 => true, + 57 => true, + 58 => true, + 59 => true, + 60 => true, + 61 => true, + _ => false, + } +} + +pub fn tdf_parquet_schema(tdf_id: u16) -> Option { + match tdf_id { + 1 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("application", DataType::UInt32, false), + Field::new( + "version", + DataType::Struct(Fields::from(vec![ + Field::new("major", DataType::UInt8, false), + Field::new("minor", DataType::UInt8, false), + Field::new("revision", DataType::UInt16, false), + Field::new("build_num", DataType::UInt32, false), + ])), + false, + ), + Field::new("kv_crc", DataType::UInt32, false), + Field::new("blocks", DataType::UInt32, false), + Field::new("uptime", DataType::UInt32, false), + Field::new("reboots", DataType::UInt16, false), + Field::new("flags", DataType::UInt8, false), + ]))), + 2 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("voltage_mv", DataType::UInt32, false), + Field::new("current_ua", DataType::Int32, false), + Field::new("soc", DataType::UInt8, false), + ]))), + 3 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("temperature", DataType::Float64, false), + Field::new("pressure", DataType::Float64, false), + Field::new("humidity", DataType::Float64, false), + ]))), + 4 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("temperature", DataType::Float64, false), + ]))), + 5 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("source", DataType::UInt8, false), + Field::new("shift", DataType::Float64, false), + ]))), + 6 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("reason", DataType::UInt8, false), + Field::new("hardware_flags", DataType::UInt32, false), + Field::new("count", DataType::UInt32, false), + Field::new("uptime", DataType::UInt32, false), + Field::new("param_1", DataType::UInt32, false), + Field::new("param_2", DataType::UInt32, false), + Field::new("thread", DataType::Utf8, false), + ]))), + 7 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("application", DataType::UInt32, false), + Field::new( + "version", + DataType::Struct(Fields::from(vec![ + Field::new("major", DataType::UInt8, false), + Field::new("minor", DataType::UInt8, false), + Field::new("revision", DataType::UInt16, false), + Field::new("build_num", DataType::UInt32, false), + ])), + false, + ), + Field::new("board_crc", DataType::UInt16, false), + Field::new("kv_crc", DataType::UInt32, false), + Field::new("blocks", DataType::UInt32, false), + Field::new("uptime", DataType::UInt32, false), + Field::new("reboots", DataType::UInt16, false), + Field::new("flags", DataType::UInt8, false), + ]))), + 8 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("temperature", DataType::Float64, false), + ]))), + 10 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 11 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 12 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 13 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 14 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 15 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 16 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 17 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 18 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "sample", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ])), + false, + ), + ]))), + 19 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "location", + DataType::Struct(Fields::from(vec![ + Field::new("latitude", DataType::Float64, false), + Field::new("longitude", DataType::Float64, false), + Field::new("height", DataType::Float64, false), + ])), + false, + ), + Field::new("h_acc", DataType::Float64, false), + Field::new("v_acc", DataType::Float64, false), + ]))), + 20 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("itow", DataType::UInt32, false), + Field::new("year", DataType::UInt16, false), + Field::new("month", DataType::UInt8, false), + Field::new("day", DataType::UInt8, false), + Field::new("hour", DataType::UInt8, false), + Field::new("min", DataType::UInt8, false), + Field::new("sec", DataType::UInt8, false), + Field::new("valid", DataType::UInt8, false), + Field::new("t_acc", DataType::UInt32, false), + Field::new("nano", DataType::Int32, false), + Field::new("fix_type", DataType::UInt8, false), + Field::new("flags", DataType::UInt8, false), + Field::new("flags2", DataType::UInt8, false), + Field::new("num_sv", DataType::UInt8, false), + Field::new("lon", DataType::Float64, false), + Field::new("lat", DataType::Float64, false), + Field::new("height", DataType::Float64, false), + Field::new("h_msl", DataType::Float64, false), + Field::new("h_acc", DataType::Float64, false), + Field::new("v_acc", DataType::Float64, false), + Field::new("vel_n", DataType::Float64, false), + Field::new("vel_e", DataType::Float64, false), + Field::new("vel_d", DataType::Float64, false), + Field::new("g_speed", DataType::Float64, false), + Field::new("head_mot", DataType::Float64, false), + Field::new("s_acc", DataType::Float64, false), + Field::new("head_acc", DataType::Float64, false), + Field::new("p_dop", DataType::Float64, false), + Field::new("flags3", DataType::UInt16, false), + Field::new( + "reserved0", + DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::UInt8, false)), 4), + false, + ), + Field::new("head_veh", DataType::Float64, false), + Field::new("mag_dec", DataType::Float64, false), + Field::new("mag_acc", DataType::Float64, false), + ]))), + 21 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "cell", + DataType::Struct(Fields::from(vec![ + Field::new("mcc", DataType::UInt16, false), + Field::new("mnc", DataType::UInt16, false), + Field::new("eci", DataType::UInt32, false), + Field::new("tac", DataType::UInt16, false), + ])), + false, + ), + Field::new("earfcn", DataType::UInt32, false), + Field::new("status", DataType::UInt8, false), + Field::new("tech", DataType::UInt8, false), + Field::new("rsrp", DataType::Float64, false), + Field::new("rsrq", DataType::Int8, false), + ]))), + 22 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "payload", + DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::UInt8, false)), 9), + false, + ), + ]))), + 23 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("count", DataType::UInt32, false), + Field::new("std_dev", DataType::UInt32, false), + ]))), + 24 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("value", DataType::UInt32, false), + ]))), + 25 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("algorithm_id", DataType::UInt32, false), + Field::new("algorithm_version", DataType::UInt16, false), + Field::new("output", DataType::Binary, false), + ]))), + 26 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("error_id", DataType::UInt32, false), + Field::new("error_ctx", DataType::UInt32, false), + ]))), + 27 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("enabled", DataType::UInt8, false), + ]))), + 28 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("time_fix", DataType::UInt16, false), + Field::new("location_fix", DataType::UInt16, false), + Field::new("num_sv", DataType::UInt8, false), + ]))), + 29 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "address", + DataType::Struct(Fields::from(vec![ + Field::new("type", DataType::UInt8, false), + Field::new("val", DataType::UInt64, false), + ])), + false, + ), + Field::new("connected", DataType::UInt8, false), + ]))), + 30 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "address", + DataType::Struct(Fields::from(vec![ + Field::new("type", DataType::UInt8, false), + Field::new("val", DataType::UInt64, false), + ])), + false, + ), + Field::new("rssi", DataType::Int8, false), + ]))), + 31 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "address", + DataType::Struct(Fields::from(vec![ + Field::new("type", DataType::UInt8, false), + Field::new("val", DataType::UInt64, false), + ])), + false, + ), + Field::new("throughput", DataType::Int32, false), + ]))), + 32 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("algorithm_id", DataType::UInt32, false), + Field::new("algorithm_version", DataType::UInt16, false), + Field::new("classes", DataType::Binary, false), + ]))), + 33 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("algorithm_id", DataType::UInt32, false), + Field::new("algorithm_version", DataType::UInt16, false), + Field::new("values", DataType::Binary, false), + ]))), + 34 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "cell", + DataType::Struct(Fields::from(vec![ + Field::new("mcc", DataType::UInt16, false), + Field::new("mnc", DataType::UInt16, false), + Field::new("eci", DataType::UInt32, false), + Field::new("tac", DataType::UInt16, false), + ])), + false, + ), + Field::new("earfcn", DataType::UInt32, false), + Field::new("rsrp", DataType::Float64, false), + Field::new("rsrq", DataType::Int8, false), + Field::new( + "neighbours", + DataType::List(Arc::new(Field::new_list_field( + DataType::Struct(Fields::from(vec![ + Field::new("earfcn", DataType::UInt32, false), + Field::new("pci", DataType::UInt16, false), + Field::new("time_diff", DataType::Float64, false), + Field::new("rsrp", DataType::Float64, false), + Field::new("rsrq", DataType::Int8, false), + ])), + false, + ))), + false, + ), + ]))), + 35 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "bssid", + DataType::Struct(Fields::from(vec![Field::new( + "val", + DataType::UInt64, + false, + )])), + false, + ), + Field::new("channel", DataType::UInt8, false), + Field::new("rsrp", DataType::Int8, false), + ]))), + 36 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("cosine", DataType::Float32, false), + ]))), + 37 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("lat", DataType::Float64, false), + Field::new("lon", DataType::Float64, false), + Field::new("height", DataType::Float64, false), + Field::new("h_acc", DataType::Float64, false), + Field::new("v_acc", DataType::Float64, false), + Field::new("h_speed", DataType::Float64, false), + Field::new("h_speed_acc", DataType::Float64, false), + Field::new("v_speed", DataType::Float64, false), + Field::new("v_speed_acc", DataType::Float64, false), + Field::new("head_mot", DataType::Float64, false), + Field::new("head_acc", DataType::Float64, false), + Field::new("year", DataType::UInt16, false), + Field::new("month", DataType::UInt8, false), + Field::new("day", DataType::UInt8, false), + Field::new("hour", DataType::UInt8, false), + Field::new("min", DataType::UInt8, false), + Field::new("sec", DataType::UInt8, false), + Field::new("ms", DataType::UInt16, false), + Field::new("p_dop", DataType::Float64, false), + Field::new("h_dop", DataType::Float64, false), + Field::new("v_dop", DataType::Float64, false), + Field::new("t_dop", DataType::Float64, false), + Field::new("flags", DataType::UInt8, false), + Field::new("num_sv", DataType::UInt8, false), + ]))), + 38 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("charge", DataType::Int32, false), + ]))), + 39 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("infuse_id", DataType::UInt64, false), + Field::new("rssi", DataType::Int8, false), + ]))), + 40 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("val", DataType::Int8, false), + ]))), + 41 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("val", DataType::Int16, false), + ]))), + 42 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("val", DataType::Int32, false), + ]))), + 43 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("timestamp", DataType::UInt32, false), + Field::new("event", DataType::Utf8, false), + ]))), + 44 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("snr", DataType::Int8, false), + Field::new("rssi", DataType::Int16, false), + Field::new("payload", DataType::Binary, false), + ]))), + 45 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("payload", DataType::Binary, false), + ]))), + 46 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("tdf_id", DataType::UInt16, false), + Field::new("frequency", DataType::UInt32, false), + ]))), + 47 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("tdf_id", DataType::UInt16, false), + Field::new("period", DataType::UInt32, false), + ]))), + 48 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "network", + DataType::Struct(Fields::from(vec![ + Field::new("bssid", DataType::UInt64, false), + Field::new("band", DataType::UInt8, false), + Field::new("channel", DataType::UInt8, false), + Field::new("iface_mode", DataType::UInt8, false), + Field::new("link_mode", DataType::UInt8, false), + Field::new("security", DataType::UInt8, false), + Field::new("rssi", DataType::Int8, false), + Field::new("beacon_interval", DataType::UInt16, false), + Field::new("twt_capable", DataType::UInt8, false), + ])), + false, + ), + ]))), + 49 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("reason", DataType::UInt8, false), + ]))), + 50 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("reason", DataType::UInt8, false), + ]))), + 51 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("num_wifi", DataType::UInt8, false), + Field::new("num_lte", DataType::UInt8, false), + ]))), + 52 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new( + "frame", + DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, false))), + false, + ), + ]))), + 53 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("voltage", DataType::UInt16, false), + ]))), + 54 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("soc", DataType::UInt8, false), + ]))), + 55 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("state", DataType::UInt8, false), + ]))), + 56 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("state", DataType::UInt8, false), + ]))), + 57 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("state", DataType::UInt8, false), + Field::new("duration", DataType::UInt32, false), + ]))), + 58 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("val", DataType::Int16, false), + ]))), + 59 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("val", DataType::Int16, false), + ]))), + 60 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("left", DataType::Int16, false), + Field::new("right", DataType::Int16, false), + ]))), + 61 => Some(Arc::new(Schema::new(vec![ + timestamp_field(), + sample_idx_field(), + Field::new("key", DataType::UInt16, false), + Field::new("value", DataType::Binary, false), + ]))), + _ => None, + } +} + +pub fn tdf_parquet_builder(tdf_id: u16, capacity: usize) -> Option { + match tdf_id { + 1 => Some(TdfParquetBatchBuilder::Tdf1Announce( + Tdf1AnnounceBuilder::new(capacity), + )), + 2 => Some(TdfParquetBatchBuilder::Tdf2BatteryState( + Tdf2BatteryStateBuilder::new(capacity), + )), + 3 => Some(TdfParquetBatchBuilder::Tdf3AmbientTempPresHum( + Tdf3AmbientTempPresHumBuilder::new(capacity), + )), + 4 => Some(TdfParquetBatchBuilder::Tdf4AmbientTemperature( + Tdf4AmbientTemperatureBuilder::new(capacity), + )), + 5 => Some(TdfParquetBatchBuilder::Tdf5TimeSync( + Tdf5TimeSyncBuilder::new(capacity), + )), + 6 => Some(TdfParquetBatchBuilder::Tdf6RebootInfo( + Tdf6RebootInfoBuilder::new(capacity), + )), + 7 => Some(TdfParquetBatchBuilder::Tdf7AnnounceV2( + Tdf7AnnounceV2Builder::new(capacity), + )), + 8 => Some(TdfParquetBatchBuilder::Tdf8SocTemperature( + Tdf8SocTemperatureBuilder::new(capacity), + )), + 10 => Some(TdfParquetBatchBuilder::Tdf10Acc2g(Tdf10Acc2gBuilder::new( + capacity, + ))), + 11 => Some(TdfParquetBatchBuilder::Tdf11Acc4g(Tdf11Acc4gBuilder::new( + capacity, + ))), + 12 => Some(TdfParquetBatchBuilder::Tdf12Acc8g(Tdf12Acc8gBuilder::new( + capacity, + ))), + 13 => Some(TdfParquetBatchBuilder::Tdf13Acc16g( + Tdf13Acc16gBuilder::new(capacity), + )), + 14 => Some(TdfParquetBatchBuilder::Tdf14Gyr125dps( + Tdf14Gyr125dpsBuilder::new(capacity), + )), + 15 => Some(TdfParquetBatchBuilder::Tdf15Gyr250dps( + Tdf15Gyr250dpsBuilder::new(capacity), + )), + 16 => Some(TdfParquetBatchBuilder::Tdf16Gyr500dps( + Tdf16Gyr500dpsBuilder::new(capacity), + )), + 17 => Some(TdfParquetBatchBuilder::Tdf17Gyr1000dps( + Tdf17Gyr1000dpsBuilder::new(capacity), + )), + 18 => Some(TdfParquetBatchBuilder::Tdf18Gyr2000dps( + Tdf18Gyr2000dpsBuilder::new(capacity), + )), + 19 => Some(TdfParquetBatchBuilder::Tdf19GcsWgs84Llha( + Tdf19GcsWgs84LlhaBuilder::new(capacity), + )), + 20 => Some(TdfParquetBatchBuilder::Tdf20UbxNavPvt( + Tdf20UbxNavPvtBuilder::new(capacity), + )), + 21 => Some(TdfParquetBatchBuilder::Tdf21LteConnStatus( + Tdf21LteConnStatusBuilder::new(capacity), + )), + 22 => Some(TdfParquetBatchBuilder::Tdf22GlobalstarPkt( + Tdf22GlobalstarPktBuilder::new(capacity), + )), + 23 => Some(TdfParquetBatchBuilder::Tdf23AccMagnitudeStdDev( + Tdf23AccMagnitudeStdDevBuilder::new(capacity), + )), + 24 => Some(TdfParquetBatchBuilder::Tdf24ActivityMetric( + Tdf24ActivityMetricBuilder::new(capacity), + )), + 25 => Some(TdfParquetBatchBuilder::Tdf25AlgorithmOutput( + Tdf25AlgorithmOutputBuilder::new(capacity), + )), + 26 => Some(TdfParquetBatchBuilder::Tdf26RuntimeError( + Tdf26RuntimeErrorBuilder::new(capacity), + )), + 27 => Some(TdfParquetBatchBuilder::Tdf27ChargerEnControl( + Tdf27ChargerEnControlBuilder::new(capacity), + )), + 28 => Some(TdfParquetBatchBuilder::Tdf28GnssFixInfo( + Tdf28GnssFixInfoBuilder::new(capacity), + )), + 29 => Some(TdfParquetBatchBuilder::Tdf29BluetoothConnection( + Tdf29BluetoothConnectionBuilder::new(capacity), + )), + 30 => Some(TdfParquetBatchBuilder::Tdf30BluetoothRssi( + Tdf30BluetoothRssiBuilder::new(capacity), + )), + 31 => Some(TdfParquetBatchBuilder::Tdf31BluetoothDataThroughput( + Tdf31BluetoothDataThroughputBuilder::new(capacity), + )), + 32 => Some(TdfParquetBatchBuilder::Tdf32AlgorithmClassHistogram( + Tdf32AlgorithmClassHistogramBuilder::new(capacity), + )), + 33 => Some(TdfParquetBatchBuilder::Tdf33AlgorithmClassTimeSeries( + Tdf33AlgorithmClassTimeSeriesBuilder::new(capacity), + )), + 34 => Some(TdfParquetBatchBuilder::Tdf34LteTacCells( + Tdf34LteTacCellsBuilder::new(capacity), + )), + 35 => Some(TdfParquetBatchBuilder::Tdf35WifiApInfo( + Tdf35WifiApInfoBuilder::new(capacity), + )), + 36 => Some(TdfParquetBatchBuilder::Tdf36DeviceTilt( + Tdf36DeviceTiltBuilder::new(capacity), + )), + 37 => Some(TdfParquetBatchBuilder::Tdf37Nrf9xGnssPvt( + Tdf37Nrf9xGnssPvtBuilder::new(capacity), + )), + 38 => Some(TdfParquetBatchBuilder::Tdf38BatteryChargeAccumulated( + Tdf38BatteryChargeAccumulatedBuilder::new(capacity), + )), + 39 => Some(TdfParquetBatchBuilder::Tdf39InfuseBluetoothRssi( + Tdf39InfuseBluetoothRssiBuilder::new(capacity), + )), + 40 => Some(TdfParquetBatchBuilder::Tdf40AdcRaw8( + Tdf40AdcRaw8Builder::new(capacity), + )), + 41 => Some(TdfParquetBatchBuilder::Tdf41AdcRaw16( + Tdf41AdcRaw16Builder::new(capacity), + )), + 42 => Some(TdfParquetBatchBuilder::Tdf42AdcRaw32( + Tdf42AdcRaw32Builder::new(capacity), + )), + 43 => Some(TdfParquetBatchBuilder::Tdf43Annotation( + Tdf43AnnotationBuilder::new(capacity), + )), + 44 => Some(TdfParquetBatchBuilder::Tdf44LoraRx( + Tdf44LoraRxBuilder::new(capacity), + )), + 45 => Some(TdfParquetBatchBuilder::Tdf45LoraTx( + Tdf45LoraTxBuilder::new(capacity), + )), + 46 => Some(TdfParquetBatchBuilder::Tdf46IdxArrayFreq( + Tdf46IdxArrayFreqBuilder::new(capacity), + )), + 47 => Some(TdfParquetBatchBuilder::Tdf47IdxArrayPeriod( + Tdf47IdxArrayPeriodBuilder::new(capacity), + )), + 48 => Some(TdfParquetBatchBuilder::Tdf48WifiConnected( + Tdf48WifiConnectedBuilder::new(capacity), + )), + 49 => Some(TdfParquetBatchBuilder::Tdf49WifiConnectionFailed( + Tdf49WifiConnectionFailedBuilder::new(capacity), + )), + 50 => Some(TdfParquetBatchBuilder::Tdf50WifiDisconnected( + Tdf50WifiDisconnectedBuilder::new(capacity), + )), + 51 => Some(TdfParquetBatchBuilder::Tdf51NetworkScanCount( + Tdf51NetworkScanCountBuilder::new(capacity), + )), + 52 => Some(TdfParquetBatchBuilder::Tdf52ExceptionStackFrame( + Tdf52ExceptionStackFrameBuilder::new(capacity), + )), + 53 => Some(TdfParquetBatchBuilder::Tdf53BatteryVoltage( + Tdf53BatteryVoltageBuilder::new(capacity), + )), + 54 => Some(TdfParquetBatchBuilder::Tdf54BatterySoc( + Tdf54BatterySocBuilder::new(capacity), + )), + 55 => Some(TdfParquetBatchBuilder::Tdf55StateEventSet( + Tdf55StateEventSetBuilder::new(capacity), + )), + 56 => Some(TdfParquetBatchBuilder::Tdf56StateEventCleared( + Tdf56StateEventClearedBuilder::new(capacity), + )), + 57 => Some(TdfParquetBatchBuilder::Tdf57StateDuration( + Tdf57StateDurationBuilder::new(capacity), + )), + 58 => Some(TdfParquetBatchBuilder::Tdf58Pcm16bitChanLeft( + Tdf58Pcm16bitChanLeftBuilder::new(capacity), + )), + 59 => Some(TdfParquetBatchBuilder::Tdf59Pcm16bitChanRight( + Tdf59Pcm16bitChanRightBuilder::new(capacity), + )), + 60 => Some(TdfParquetBatchBuilder::Tdf60Pcm16bitChanDual( + Tdf60Pcm16bitChanDualBuilder::new(capacity), + )), + 61 => Some(TdfParquetBatchBuilder::Tdf61KvsValueChanged( + Tdf61KvsValueChangedBuilder::new(capacity), + )), + _ => None, + } +} + +#[derive(Clone, Copy, Debug)] +pub struct TdfParquetRowMeta { + pub time_unix_micros: Option, + pub sample_idx: Option, +} + +pub enum TdfParquetBatchBuilder { + Tdf1Announce(Tdf1AnnounceBuilder), + Tdf2BatteryState(Tdf2BatteryStateBuilder), + Tdf3AmbientTempPresHum(Tdf3AmbientTempPresHumBuilder), + Tdf4AmbientTemperature(Tdf4AmbientTemperatureBuilder), + Tdf5TimeSync(Tdf5TimeSyncBuilder), + Tdf6RebootInfo(Tdf6RebootInfoBuilder), + Tdf7AnnounceV2(Tdf7AnnounceV2Builder), + Tdf8SocTemperature(Tdf8SocTemperatureBuilder), + Tdf10Acc2g(Tdf10Acc2gBuilder), + Tdf11Acc4g(Tdf11Acc4gBuilder), + Tdf12Acc8g(Tdf12Acc8gBuilder), + Tdf13Acc16g(Tdf13Acc16gBuilder), + Tdf14Gyr125dps(Tdf14Gyr125dpsBuilder), + Tdf15Gyr250dps(Tdf15Gyr250dpsBuilder), + Tdf16Gyr500dps(Tdf16Gyr500dpsBuilder), + Tdf17Gyr1000dps(Tdf17Gyr1000dpsBuilder), + Tdf18Gyr2000dps(Tdf18Gyr2000dpsBuilder), + Tdf19GcsWgs84Llha(Tdf19GcsWgs84LlhaBuilder), + Tdf20UbxNavPvt(Tdf20UbxNavPvtBuilder), + Tdf21LteConnStatus(Tdf21LteConnStatusBuilder), + Tdf22GlobalstarPkt(Tdf22GlobalstarPktBuilder), + Tdf23AccMagnitudeStdDev(Tdf23AccMagnitudeStdDevBuilder), + Tdf24ActivityMetric(Tdf24ActivityMetricBuilder), + Tdf25AlgorithmOutput(Tdf25AlgorithmOutputBuilder), + Tdf26RuntimeError(Tdf26RuntimeErrorBuilder), + Tdf27ChargerEnControl(Tdf27ChargerEnControlBuilder), + Tdf28GnssFixInfo(Tdf28GnssFixInfoBuilder), + Tdf29BluetoothConnection(Tdf29BluetoothConnectionBuilder), + Tdf30BluetoothRssi(Tdf30BluetoothRssiBuilder), + Tdf31BluetoothDataThroughput(Tdf31BluetoothDataThroughputBuilder), + Tdf32AlgorithmClassHistogram(Tdf32AlgorithmClassHistogramBuilder), + Tdf33AlgorithmClassTimeSeries(Tdf33AlgorithmClassTimeSeriesBuilder), + Tdf34LteTacCells(Tdf34LteTacCellsBuilder), + Tdf35WifiApInfo(Tdf35WifiApInfoBuilder), + Tdf36DeviceTilt(Tdf36DeviceTiltBuilder), + Tdf37Nrf9xGnssPvt(Tdf37Nrf9xGnssPvtBuilder), + Tdf38BatteryChargeAccumulated(Tdf38BatteryChargeAccumulatedBuilder), + Tdf39InfuseBluetoothRssi(Tdf39InfuseBluetoothRssiBuilder), + Tdf40AdcRaw8(Tdf40AdcRaw8Builder), + Tdf41AdcRaw16(Tdf41AdcRaw16Builder), + Tdf42AdcRaw32(Tdf42AdcRaw32Builder), + Tdf43Annotation(Tdf43AnnotationBuilder), + Tdf44LoraRx(Tdf44LoraRxBuilder), + Tdf45LoraTx(Tdf45LoraTxBuilder), + Tdf46IdxArrayFreq(Tdf46IdxArrayFreqBuilder), + Tdf47IdxArrayPeriod(Tdf47IdxArrayPeriodBuilder), + Tdf48WifiConnected(Tdf48WifiConnectedBuilder), + Tdf49WifiConnectionFailed(Tdf49WifiConnectionFailedBuilder), + Tdf50WifiDisconnected(Tdf50WifiDisconnectedBuilder), + Tdf51NetworkScanCount(Tdf51NetworkScanCountBuilder), + Tdf52ExceptionStackFrame(Tdf52ExceptionStackFrameBuilder), + Tdf53BatteryVoltage(Tdf53BatteryVoltageBuilder), + Tdf54BatterySoc(Tdf54BatterySocBuilder), + Tdf55StateEventSet(Tdf55StateEventSetBuilder), + Tdf56StateEventCleared(Tdf56StateEventClearedBuilder), + Tdf57StateDuration(Tdf57StateDurationBuilder), + Tdf58Pcm16bitChanLeft(Tdf58Pcm16bitChanLeftBuilder), + Tdf59Pcm16bitChanRight(Tdf59Pcm16bitChanRightBuilder), + Tdf60Pcm16bitChanDual(Tdf60Pcm16bitChanDualBuilder), + Tdf61KvsValueChanged(Tdf61KvsValueChangedBuilder), +} + +impl TdfParquetBatchBuilder { + pub fn schema(&self) -> SchemaRef { + match self { + Self::Tdf1Announce(builder) => builder.schema(), + Self::Tdf2BatteryState(builder) => builder.schema(), + Self::Tdf3AmbientTempPresHum(builder) => builder.schema(), + Self::Tdf4AmbientTemperature(builder) => builder.schema(), + Self::Tdf5TimeSync(builder) => builder.schema(), + Self::Tdf6RebootInfo(builder) => builder.schema(), + Self::Tdf7AnnounceV2(builder) => builder.schema(), + Self::Tdf8SocTemperature(builder) => builder.schema(), + Self::Tdf10Acc2g(builder) => builder.schema(), + Self::Tdf11Acc4g(builder) => builder.schema(), + Self::Tdf12Acc8g(builder) => builder.schema(), + Self::Tdf13Acc16g(builder) => builder.schema(), + Self::Tdf14Gyr125dps(builder) => builder.schema(), + Self::Tdf15Gyr250dps(builder) => builder.schema(), + Self::Tdf16Gyr500dps(builder) => builder.schema(), + Self::Tdf17Gyr1000dps(builder) => builder.schema(), + Self::Tdf18Gyr2000dps(builder) => builder.schema(), + Self::Tdf19GcsWgs84Llha(builder) => builder.schema(), + Self::Tdf20UbxNavPvt(builder) => builder.schema(), + Self::Tdf21LteConnStatus(builder) => builder.schema(), + Self::Tdf22GlobalstarPkt(builder) => builder.schema(), + Self::Tdf23AccMagnitudeStdDev(builder) => builder.schema(), + Self::Tdf24ActivityMetric(builder) => builder.schema(), + Self::Tdf25AlgorithmOutput(builder) => builder.schema(), + Self::Tdf26RuntimeError(builder) => builder.schema(), + Self::Tdf27ChargerEnControl(builder) => builder.schema(), + Self::Tdf28GnssFixInfo(builder) => builder.schema(), + Self::Tdf29BluetoothConnection(builder) => builder.schema(), + Self::Tdf30BluetoothRssi(builder) => builder.schema(), + Self::Tdf31BluetoothDataThroughput(builder) => builder.schema(), + Self::Tdf32AlgorithmClassHistogram(builder) => builder.schema(), + Self::Tdf33AlgorithmClassTimeSeries(builder) => builder.schema(), + Self::Tdf34LteTacCells(builder) => builder.schema(), + Self::Tdf35WifiApInfo(builder) => builder.schema(), + Self::Tdf36DeviceTilt(builder) => builder.schema(), + Self::Tdf37Nrf9xGnssPvt(builder) => builder.schema(), + Self::Tdf38BatteryChargeAccumulated(builder) => builder.schema(), + Self::Tdf39InfuseBluetoothRssi(builder) => builder.schema(), + Self::Tdf40AdcRaw8(builder) => builder.schema(), + Self::Tdf41AdcRaw16(builder) => builder.schema(), + Self::Tdf42AdcRaw32(builder) => builder.schema(), + Self::Tdf43Annotation(builder) => builder.schema(), + Self::Tdf44LoraRx(builder) => builder.schema(), + Self::Tdf45LoraTx(builder) => builder.schema(), + Self::Tdf46IdxArrayFreq(builder) => builder.schema(), + Self::Tdf47IdxArrayPeriod(builder) => builder.schema(), + Self::Tdf48WifiConnected(builder) => builder.schema(), + Self::Tdf49WifiConnectionFailed(builder) => builder.schema(), + Self::Tdf50WifiDisconnected(builder) => builder.schema(), + Self::Tdf51NetworkScanCount(builder) => builder.schema(), + Self::Tdf52ExceptionStackFrame(builder) => builder.schema(), + Self::Tdf53BatteryVoltage(builder) => builder.schema(), + Self::Tdf54BatterySoc(builder) => builder.schema(), + Self::Tdf55StateEventSet(builder) => builder.schema(), + Self::Tdf56StateEventCleared(builder) => builder.schema(), + Self::Tdf57StateDuration(builder) => builder.schema(), + Self::Tdf58Pcm16bitChanLeft(builder) => builder.schema(), + Self::Tdf59Pcm16bitChanRight(builder) => builder.schema(), + Self::Tdf60Pcm16bitChanDual(builder) => builder.schema(), + Self::Tdf61KvsValueChanged(builder) => builder.schema(), + } + } + + pub fn rows(&self) -> usize { + match self { + Self::Tdf1Announce(builder) => builder.rows(), + Self::Tdf2BatteryState(builder) => builder.rows(), + Self::Tdf3AmbientTempPresHum(builder) => builder.rows(), + Self::Tdf4AmbientTemperature(builder) => builder.rows(), + Self::Tdf5TimeSync(builder) => builder.rows(), + Self::Tdf6RebootInfo(builder) => builder.rows(), + Self::Tdf7AnnounceV2(builder) => builder.rows(), + Self::Tdf8SocTemperature(builder) => builder.rows(), + Self::Tdf10Acc2g(builder) => builder.rows(), + Self::Tdf11Acc4g(builder) => builder.rows(), + Self::Tdf12Acc8g(builder) => builder.rows(), + Self::Tdf13Acc16g(builder) => builder.rows(), + Self::Tdf14Gyr125dps(builder) => builder.rows(), + Self::Tdf15Gyr250dps(builder) => builder.rows(), + Self::Tdf16Gyr500dps(builder) => builder.rows(), + Self::Tdf17Gyr1000dps(builder) => builder.rows(), + Self::Tdf18Gyr2000dps(builder) => builder.rows(), + Self::Tdf19GcsWgs84Llha(builder) => builder.rows(), + Self::Tdf20UbxNavPvt(builder) => builder.rows(), + Self::Tdf21LteConnStatus(builder) => builder.rows(), + Self::Tdf22GlobalstarPkt(builder) => builder.rows(), + Self::Tdf23AccMagnitudeStdDev(builder) => builder.rows(), + Self::Tdf24ActivityMetric(builder) => builder.rows(), + Self::Tdf25AlgorithmOutput(builder) => builder.rows(), + Self::Tdf26RuntimeError(builder) => builder.rows(), + Self::Tdf27ChargerEnControl(builder) => builder.rows(), + Self::Tdf28GnssFixInfo(builder) => builder.rows(), + Self::Tdf29BluetoothConnection(builder) => builder.rows(), + Self::Tdf30BluetoothRssi(builder) => builder.rows(), + Self::Tdf31BluetoothDataThroughput(builder) => builder.rows(), + Self::Tdf32AlgorithmClassHistogram(builder) => builder.rows(), + Self::Tdf33AlgorithmClassTimeSeries(builder) => builder.rows(), + Self::Tdf34LteTacCells(builder) => builder.rows(), + Self::Tdf35WifiApInfo(builder) => builder.rows(), + Self::Tdf36DeviceTilt(builder) => builder.rows(), + Self::Tdf37Nrf9xGnssPvt(builder) => builder.rows(), + Self::Tdf38BatteryChargeAccumulated(builder) => builder.rows(), + Self::Tdf39InfuseBluetoothRssi(builder) => builder.rows(), + Self::Tdf40AdcRaw8(builder) => builder.rows(), + Self::Tdf41AdcRaw16(builder) => builder.rows(), + Self::Tdf42AdcRaw32(builder) => builder.rows(), + Self::Tdf43Annotation(builder) => builder.rows(), + Self::Tdf44LoraRx(builder) => builder.rows(), + Self::Tdf45LoraTx(builder) => builder.rows(), + Self::Tdf46IdxArrayFreq(builder) => builder.rows(), + Self::Tdf47IdxArrayPeriod(builder) => builder.rows(), + Self::Tdf48WifiConnected(builder) => builder.rows(), + Self::Tdf49WifiConnectionFailed(builder) => builder.rows(), + Self::Tdf50WifiDisconnected(builder) => builder.rows(), + Self::Tdf51NetworkScanCount(builder) => builder.rows(), + Self::Tdf52ExceptionStackFrame(builder) => builder.rows(), + Self::Tdf53BatteryVoltage(builder) => builder.rows(), + Self::Tdf54BatterySoc(builder) => builder.rows(), + Self::Tdf55StateEventSet(builder) => builder.rows(), + Self::Tdf56StateEventCleared(builder) => builder.rows(), + Self::Tdf57StateDuration(builder) => builder.rows(), + Self::Tdf58Pcm16bitChanLeft(builder) => builder.rows(), + Self::Tdf59Pcm16bitChanRight(builder) => builder.rows(), + Self::Tdf60Pcm16bitChanDual(builder) => builder.rows(), + Self::Tdf61KvsValueChanged(builder) => builder.rows(), + } + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + match self { + Self::Tdf1Announce(builder) => builder.append(meta, size, cursor), + Self::Tdf2BatteryState(builder) => builder.append(meta, size, cursor), + Self::Tdf3AmbientTempPresHum(builder) => builder.append(meta, size, cursor), + Self::Tdf4AmbientTemperature(builder) => builder.append(meta, size, cursor), + Self::Tdf5TimeSync(builder) => builder.append(meta, size, cursor), + Self::Tdf6RebootInfo(builder) => builder.append(meta, size, cursor), + Self::Tdf7AnnounceV2(builder) => builder.append(meta, size, cursor), + Self::Tdf8SocTemperature(builder) => builder.append(meta, size, cursor), + Self::Tdf10Acc2g(builder) => builder.append(meta, size, cursor), + Self::Tdf11Acc4g(builder) => builder.append(meta, size, cursor), + Self::Tdf12Acc8g(builder) => builder.append(meta, size, cursor), + Self::Tdf13Acc16g(builder) => builder.append(meta, size, cursor), + Self::Tdf14Gyr125dps(builder) => builder.append(meta, size, cursor), + Self::Tdf15Gyr250dps(builder) => builder.append(meta, size, cursor), + Self::Tdf16Gyr500dps(builder) => builder.append(meta, size, cursor), + Self::Tdf17Gyr1000dps(builder) => builder.append(meta, size, cursor), + Self::Tdf18Gyr2000dps(builder) => builder.append(meta, size, cursor), + Self::Tdf19GcsWgs84Llha(builder) => builder.append(meta, size, cursor), + Self::Tdf20UbxNavPvt(builder) => builder.append(meta, size, cursor), + Self::Tdf21LteConnStatus(builder) => builder.append(meta, size, cursor), + Self::Tdf22GlobalstarPkt(builder) => builder.append(meta, size, cursor), + Self::Tdf23AccMagnitudeStdDev(builder) => builder.append(meta, size, cursor), + Self::Tdf24ActivityMetric(builder) => builder.append(meta, size, cursor), + Self::Tdf25AlgorithmOutput(builder) => builder.append(meta, size, cursor), + Self::Tdf26RuntimeError(builder) => builder.append(meta, size, cursor), + Self::Tdf27ChargerEnControl(builder) => builder.append(meta, size, cursor), + Self::Tdf28GnssFixInfo(builder) => builder.append(meta, size, cursor), + Self::Tdf29BluetoothConnection(builder) => builder.append(meta, size, cursor), + Self::Tdf30BluetoothRssi(builder) => builder.append(meta, size, cursor), + Self::Tdf31BluetoothDataThroughput(builder) => builder.append(meta, size, cursor), + Self::Tdf32AlgorithmClassHistogram(builder) => builder.append(meta, size, cursor), + Self::Tdf33AlgorithmClassTimeSeries(builder) => builder.append(meta, size, cursor), + Self::Tdf34LteTacCells(builder) => builder.append(meta, size, cursor), + Self::Tdf35WifiApInfo(builder) => builder.append(meta, size, cursor), + Self::Tdf36DeviceTilt(builder) => builder.append(meta, size, cursor), + Self::Tdf37Nrf9xGnssPvt(builder) => builder.append(meta, size, cursor), + Self::Tdf38BatteryChargeAccumulated(builder) => builder.append(meta, size, cursor), + Self::Tdf39InfuseBluetoothRssi(builder) => builder.append(meta, size, cursor), + Self::Tdf40AdcRaw8(builder) => builder.append(meta, size, cursor), + Self::Tdf41AdcRaw16(builder) => builder.append(meta, size, cursor), + Self::Tdf42AdcRaw32(builder) => builder.append(meta, size, cursor), + Self::Tdf43Annotation(builder) => builder.append(meta, size, cursor), + Self::Tdf44LoraRx(builder) => builder.append(meta, size, cursor), + Self::Tdf45LoraTx(builder) => builder.append(meta, size, cursor), + Self::Tdf46IdxArrayFreq(builder) => builder.append(meta, size, cursor), + Self::Tdf47IdxArrayPeriod(builder) => builder.append(meta, size, cursor), + Self::Tdf48WifiConnected(builder) => builder.append(meta, size, cursor), + Self::Tdf49WifiConnectionFailed(builder) => builder.append(meta, size, cursor), + Self::Tdf50WifiDisconnected(builder) => builder.append(meta, size, cursor), + Self::Tdf51NetworkScanCount(builder) => builder.append(meta, size, cursor), + Self::Tdf52ExceptionStackFrame(builder) => builder.append(meta, size, cursor), + Self::Tdf53BatteryVoltage(builder) => builder.append(meta, size, cursor), + Self::Tdf54BatterySoc(builder) => builder.append(meta, size, cursor), + Self::Tdf55StateEventSet(builder) => builder.append(meta, size, cursor), + Self::Tdf56StateEventCleared(builder) => builder.append(meta, size, cursor), + Self::Tdf57StateDuration(builder) => builder.append(meta, size, cursor), + Self::Tdf58Pcm16bitChanLeft(builder) => builder.append(meta, size, cursor), + Self::Tdf59Pcm16bitChanRight(builder) => builder.append(meta, size, cursor), + Self::Tdf60Pcm16bitChanDual(builder) => builder.append(meta, size, cursor), + Self::Tdf61KvsValueChanged(builder) => builder.append(meta, size, cursor), + } + } + + pub fn finish_batch(&mut self) -> std::result::Result { + match self { + Self::Tdf1Announce(builder) => builder.finish_batch(), + Self::Tdf2BatteryState(builder) => builder.finish_batch(), + Self::Tdf3AmbientTempPresHum(builder) => builder.finish_batch(), + Self::Tdf4AmbientTemperature(builder) => builder.finish_batch(), + Self::Tdf5TimeSync(builder) => builder.finish_batch(), + Self::Tdf6RebootInfo(builder) => builder.finish_batch(), + Self::Tdf7AnnounceV2(builder) => builder.finish_batch(), + Self::Tdf8SocTemperature(builder) => builder.finish_batch(), + Self::Tdf10Acc2g(builder) => builder.finish_batch(), + Self::Tdf11Acc4g(builder) => builder.finish_batch(), + Self::Tdf12Acc8g(builder) => builder.finish_batch(), + Self::Tdf13Acc16g(builder) => builder.finish_batch(), + Self::Tdf14Gyr125dps(builder) => builder.finish_batch(), + Self::Tdf15Gyr250dps(builder) => builder.finish_batch(), + Self::Tdf16Gyr500dps(builder) => builder.finish_batch(), + Self::Tdf17Gyr1000dps(builder) => builder.finish_batch(), + Self::Tdf18Gyr2000dps(builder) => builder.finish_batch(), + Self::Tdf19GcsWgs84Llha(builder) => builder.finish_batch(), + Self::Tdf20UbxNavPvt(builder) => builder.finish_batch(), + Self::Tdf21LteConnStatus(builder) => builder.finish_batch(), + Self::Tdf22GlobalstarPkt(builder) => builder.finish_batch(), + Self::Tdf23AccMagnitudeStdDev(builder) => builder.finish_batch(), + Self::Tdf24ActivityMetric(builder) => builder.finish_batch(), + Self::Tdf25AlgorithmOutput(builder) => builder.finish_batch(), + Self::Tdf26RuntimeError(builder) => builder.finish_batch(), + Self::Tdf27ChargerEnControl(builder) => builder.finish_batch(), + Self::Tdf28GnssFixInfo(builder) => builder.finish_batch(), + Self::Tdf29BluetoothConnection(builder) => builder.finish_batch(), + Self::Tdf30BluetoothRssi(builder) => builder.finish_batch(), + Self::Tdf31BluetoothDataThroughput(builder) => builder.finish_batch(), + Self::Tdf32AlgorithmClassHistogram(builder) => builder.finish_batch(), + Self::Tdf33AlgorithmClassTimeSeries(builder) => builder.finish_batch(), + Self::Tdf34LteTacCells(builder) => builder.finish_batch(), + Self::Tdf35WifiApInfo(builder) => builder.finish_batch(), + Self::Tdf36DeviceTilt(builder) => builder.finish_batch(), + Self::Tdf37Nrf9xGnssPvt(builder) => builder.finish_batch(), + Self::Tdf38BatteryChargeAccumulated(builder) => builder.finish_batch(), + Self::Tdf39InfuseBluetoothRssi(builder) => builder.finish_batch(), + Self::Tdf40AdcRaw8(builder) => builder.finish_batch(), + Self::Tdf41AdcRaw16(builder) => builder.finish_batch(), + Self::Tdf42AdcRaw32(builder) => builder.finish_batch(), + Self::Tdf43Annotation(builder) => builder.finish_batch(), + Self::Tdf44LoraRx(builder) => builder.finish_batch(), + Self::Tdf45LoraTx(builder) => builder.finish_batch(), + Self::Tdf46IdxArrayFreq(builder) => builder.finish_batch(), + Self::Tdf47IdxArrayPeriod(builder) => builder.finish_batch(), + Self::Tdf48WifiConnected(builder) => builder.finish_batch(), + Self::Tdf49WifiConnectionFailed(builder) => builder.finish_batch(), + Self::Tdf50WifiDisconnected(builder) => builder.finish_batch(), + Self::Tdf51NetworkScanCount(builder) => builder.finish_batch(), + Self::Tdf52ExceptionStackFrame(builder) => builder.finish_batch(), + Self::Tdf53BatteryVoltage(builder) => builder.finish_batch(), + Self::Tdf54BatterySoc(builder) => builder.finish_batch(), + Self::Tdf55StateEventSet(builder) => builder.finish_batch(), + Self::Tdf56StateEventCleared(builder) => builder.finish_batch(), + Self::Tdf57StateDuration(builder) => builder.finish_batch(), + Self::Tdf58Pcm16bitChanLeft(builder) => builder.finish_batch(), + Self::Tdf59Pcm16bitChanRight(builder) => builder.finish_batch(), + Self::Tdf60Pcm16bitChanDual(builder) => builder.finish_batch(), + Self::Tdf61KvsValueChanged(builder) => builder.finish_batch(), + } + } +} + +pub struct Tdf1AnnounceBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + application: Vec, + version_major: Vec, + version_minor: Vec, + version_revision: Vec, + version_build_num: Vec, + kv_crc: Vec, + blocks: Vec, + uptime: Vec, + reboots: Vec, + flags: Vec, +} + +impl Tdf1AnnounceBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + application: Vec::with_capacity(capacity), + version_major: Vec::with_capacity(capacity), + version_minor: Vec::with_capacity(capacity), + version_revision: Vec::with_capacity(capacity), + version_build_num: Vec::with_capacity(capacity), + kv_crc: Vec::with_capacity(capacity), + blocks: Vec::with_capacity(capacity), + uptime: Vec::with_capacity(capacity), + reboots: Vec::with_capacity(capacity), + flags: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(1).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.application.push(cursor.read_u32::()?); + self.version_major.push(cursor.read_u8()?); + self.version_minor.push(cursor.read_u8()?); + self.version_revision + .push(cursor.read_u16::()?); + self.version_build_num + .push(cursor.read_u32::()?); + self.kv_crc.push(cursor.read_u32::()?); + self.blocks.push(cursor.read_u32::()?); + self.uptime.push(cursor.read_u32::()?); + self.reboots.push(cursor.read_u16::()?); + self.flags.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.application))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("major", DataType::UInt8, false), + Field::new("minor", DataType::UInt8, false), + Field::new("revision", DataType::UInt16, false), + Field::new("build_num", DataType::UInt32, false), + ]), + vec![ + Arc::new(UInt8Array::from(std::mem::take(&mut self.version_major))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.version_minor))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take( + &mut self.version_revision, + ))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take( + &mut self.version_build_num, + ))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.kv_crc))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.blocks))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.uptime))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.reboots))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.flags))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf2BatteryStateBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + voltage_mv: Vec, + current_ua: Vec, + soc: Vec, +} + +impl Tdf2BatteryStateBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + voltage_mv: Vec::with_capacity(capacity), + current_ua: Vec::with_capacity(capacity), + soc: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(2).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.voltage_mv.push(cursor.read_u32::()?); + self.current_ua.push(cursor.read_i32::()?); + self.soc.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.voltage_mv))) as ArrayRef, + Arc::new(Int32Array::from(std::mem::take(&mut self.current_ua))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.soc))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf3AmbientTempPresHumBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + temperature: Vec, + pressure: Vec, + humidity: Vec, +} + +impl Tdf3AmbientTempPresHumBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + temperature: Vec::with_capacity(capacity), + pressure: Vec::with_capacity(capacity), + humidity: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(3).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.temperature + .push(cursor.read_i32::()? as f64 / 1000.0); + self.pressure + .push(cursor.read_u32::()? as f64 / 1000.0); + self.humidity + .push(cursor.read_u16::()? as f64 / 100.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.temperature))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.pressure))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.humidity))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf4AmbientTemperatureBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + temperature: Vec, +} + +impl Tdf4AmbientTemperatureBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + temperature: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(4).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.temperature + .push(cursor.read_i32::()? as f64 / 1000.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.temperature))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf5TimeSyncBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + source: Vec, + shift: Vec, +} + +impl Tdf5TimeSyncBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + source: Vec::with_capacity(capacity), + shift: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(5).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.source.push(cursor.read_u8()?); + self.shift + .push(cursor.read_i32::()? as f64 / 1000000.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.source))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.shift))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf6RebootInfoBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + reason: Vec, + hardware_flags: Vec, + count: Vec, + uptime: Vec, + param_1: Vec, + param_2: Vec, + thread: Vec, +} + +impl Tdf6RebootInfoBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + reason: Vec::with_capacity(capacity), + hardware_flags: Vec::with_capacity(capacity), + count: Vec::with_capacity(capacity), + uptime: Vec::with_capacity(capacity), + param_1: Vec::with_capacity(capacity), + param_2: Vec::with_capacity(capacity), + thread: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(6).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.reason.push(cursor.read_u8()?); + self.hardware_flags.push(cursor.read_u32::()?); + self.count.push(cursor.read_u32::()?); + self.uptime.push(cursor.read_u32::()?); + self.param_1.push(cursor.read_u32::()?); + self.param_2.push(cursor.read_u32::()?); + self.thread.push(tdf_field_read_string_to_string( + cursor, + cursor_start, + 8, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.reason))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.hardware_flags))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.count))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.uptime))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.param_1))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.param_2))) as ArrayRef, + Arc::new(StringArray::from_iter_values(std::mem::take( + &mut self.thread, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf7AnnounceV2Builder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + application: Vec, + version_major: Vec, + version_minor: Vec, + version_revision: Vec, + version_build_num: Vec, + board_crc: Vec, + kv_crc: Vec, + blocks: Vec, + uptime: Vec, + reboots: Vec, + flags: Vec, +} + +impl Tdf7AnnounceV2Builder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + application: Vec::with_capacity(capacity), + version_major: Vec::with_capacity(capacity), + version_minor: Vec::with_capacity(capacity), + version_revision: Vec::with_capacity(capacity), + version_build_num: Vec::with_capacity(capacity), + board_crc: Vec::with_capacity(capacity), + kv_crc: Vec::with_capacity(capacity), + blocks: Vec::with_capacity(capacity), + uptime: Vec::with_capacity(capacity), + reboots: Vec::with_capacity(capacity), + flags: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(7).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.application.push(cursor.read_u32::()?); + self.version_major.push(cursor.read_u8()?); + self.version_minor.push(cursor.read_u8()?); + self.version_revision + .push(cursor.read_u16::()?); + self.version_build_num + .push(cursor.read_u32::()?); + self.board_crc.push(cursor.read_u16::()?); + self.kv_crc.push(cursor.read_u32::()?); + self.blocks.push(cursor.read_u32::()?); + self.uptime.push(cursor.read_u32::()?); + self.reboots.push(cursor.read_u16::()?); + self.flags.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.application))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("major", DataType::UInt8, false), + Field::new("minor", DataType::UInt8, false), + Field::new("revision", DataType::UInt16, false), + Field::new("build_num", DataType::UInt32, false), + ]), + vec![ + Arc::new(UInt8Array::from(std::mem::take(&mut self.version_major))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.version_minor))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take( + &mut self.version_revision, + ))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take( + &mut self.version_build_num, + ))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.board_crc))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.kv_crc))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.blocks))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.uptime))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.reboots))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.flags))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf8SocTemperatureBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + temperature: Vec, +} + +impl Tdf8SocTemperatureBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + temperature: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(8).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.temperature + .push(cursor.read_i16::()? as f64 / 100.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.temperature))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf10Acc2gBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf10Acc2gBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(10).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf11Acc4gBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf11Acc4gBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(11).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf12Acc8gBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf12Acc8gBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(12).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf13Acc16gBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf13Acc16gBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(13).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf14Gyr125dpsBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf14Gyr125dpsBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(14).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf15Gyr250dpsBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf15Gyr250dpsBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(15).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf16Gyr500dpsBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf16Gyr500dpsBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(16).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf17Gyr1000dpsBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf17Gyr1000dpsBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(17).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf18Gyr2000dpsBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + sample_x: Vec, + sample_y: Vec, + sample_z: Vec, +} + +impl Tdf18Gyr2000dpsBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + sample_x: Vec::with_capacity(capacity), + sample_y: Vec::with_capacity(capacity), + sample_z: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(18).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.sample_x.push(cursor.read_i16::()?); + self.sample_y.push(cursor.read_i16::()?); + self.sample_z.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("x", DataType::Int16, false), + Field::new("y", DataType::Int16, false), + Field::new("z", DataType::Int16, false), + ]), + vec![ + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_x))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_y))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.sample_z))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf19GcsWgs84LlhaBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + location_latitude: Vec, + location_longitude: Vec, + location_height: Vec, + h_acc: Vec, + v_acc: Vec, +} + +impl Tdf19GcsWgs84LlhaBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + location_latitude: Vec::with_capacity(capacity), + location_longitude: Vec::with_capacity(capacity), + location_height: Vec::with_capacity(capacity), + h_acc: Vec::with_capacity(capacity), + v_acc: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(19).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.location_latitude + .push(cursor.read_i32::()? as f64 / 10000000.0); + self.location_longitude + .push(cursor.read_i32::()? as f64 / 10000000.0); + self.location_height + .push(cursor.read_i32::()? as f64 / 1000.0); + self.h_acc + .push(cursor.read_i32::()? as f64 / 1000.0); + self.v_acc + .push(cursor.read_i32::()? as f64 / 1000.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("latitude", DataType::Float64, false), + Field::new("longitude", DataType::Float64, false), + Field::new("height", DataType::Float64, false), + ]), + vec![ + Arc::new(Float64Array::from(std::mem::take( + &mut self.location_latitude, + ))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take( + &mut self.location_longitude, + ))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take( + &mut self.location_height, + ))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.v_acc))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf20UbxNavPvtBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + itow: Vec, + year: Vec, + month: Vec, + day: Vec, + hour: Vec, + min: Vec, + sec: Vec, + valid: Vec, + t_acc: Vec, + nano: Vec, + fix_type: Vec, + flags: Vec, + flags2: Vec, + num_sv: Vec, + lon: Vec, + lat: Vec, + height: Vec, + h_msl: Vec, + h_acc: Vec, + v_acc: Vec, + vel_n: Vec, + vel_e: Vec, + vel_d: Vec, + g_speed: Vec, + head_mot: Vec, + s_acc: Vec, + head_acc: Vec, + p_dop: Vec, + flags3: Vec, + reserved0: Vec, + head_veh: Vec, + mag_dec: Vec, + mag_acc: Vec, +} + +impl Tdf20UbxNavPvtBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + itow: Vec::with_capacity(capacity), + year: Vec::with_capacity(capacity), + month: Vec::with_capacity(capacity), + day: Vec::with_capacity(capacity), + hour: Vec::with_capacity(capacity), + min: Vec::with_capacity(capacity), + sec: Vec::with_capacity(capacity), + valid: Vec::with_capacity(capacity), + t_acc: Vec::with_capacity(capacity), + nano: Vec::with_capacity(capacity), + fix_type: Vec::with_capacity(capacity), + flags: Vec::with_capacity(capacity), + flags2: Vec::with_capacity(capacity), + num_sv: Vec::with_capacity(capacity), + lon: Vec::with_capacity(capacity), + lat: Vec::with_capacity(capacity), + height: Vec::with_capacity(capacity), + h_msl: Vec::with_capacity(capacity), + h_acc: Vec::with_capacity(capacity), + v_acc: Vec::with_capacity(capacity), + vel_n: Vec::with_capacity(capacity), + vel_e: Vec::with_capacity(capacity), + vel_d: Vec::with_capacity(capacity), + g_speed: Vec::with_capacity(capacity), + head_mot: Vec::with_capacity(capacity), + s_acc: Vec::with_capacity(capacity), + head_acc: Vec::with_capacity(capacity), + p_dop: Vec::with_capacity(capacity), + flags3: Vec::with_capacity(capacity), + reserved0: Vec::with_capacity(capacity * 4), + head_veh: Vec::with_capacity(capacity), + mag_dec: Vec::with_capacity(capacity), + mag_acc: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(20).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.itow.push(cursor.read_u32::()?); + self.year.push(cursor.read_u16::()?); + self.month.push(cursor.read_u8()?); + self.day.push(cursor.read_u8()?); + self.hour.push(cursor.read_u8()?); + self.min.push(cursor.read_u8()?); + self.sec.push(cursor.read_u8()?); + self.valid.push(cursor.read_u8()?); + self.t_acc.push(cursor.read_u32::()?); + self.nano.push(cursor.read_i32::()?); + self.fix_type.push(cursor.read_u8()?); + self.flags.push(cursor.read_u8()?); + self.flags2.push(cursor.read_u8()?); + self.num_sv.push(cursor.read_u8()?); + self.lon + .push(cursor.read_i32::()? as f64 / 10000000.0); + self.lat + .push(cursor.read_i32::()? as f64 / 10000000.0); + self.height + .push(cursor.read_i32::()? as f64 / 1000.0); + self.h_msl + .push(cursor.read_i32::()? as f64 / 1000.0); + self.h_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.v_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.vel_n + .push(cursor.read_i32::()? as f64 / 1000.0); + self.vel_e + .push(cursor.read_i32::()? as f64 / 1000.0); + self.vel_d + .push(cursor.read_i32::()? as f64 / 1000.0); + self.g_speed + .push(cursor.read_i32::()? as f64 / 1000.0); + self.head_mot + .push(cursor.read_i32::()? as f64 / 100000.0); + self.s_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.head_acc + .push(cursor.read_u32::()? as f64 / 100000.0); + self.p_dop + .push(cursor.read_u16::()? as f64 / 100.0); + self.flags3.push(cursor.read_u16::()?); + self.reserved0.push(cursor.read_u8()?); + self.reserved0.push(cursor.read_u8()?); + self.reserved0.push(cursor.read_u8()?); + self.reserved0.push(cursor.read_u8()?); + self.head_veh + .push(cursor.read_i32::()? as f64 / 100000.0); + self.mag_dec + .push(cursor.read_i16::()? as f64 / 100.0); + self.mag_acc + .push(cursor.read_u16::()? as f64 / 100.0); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.itow))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.year))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.month))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.day))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.hour))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.min))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.sec))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.valid))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.t_acc))) as ArrayRef, + Arc::new(Int32Array::from(std::mem::take(&mut self.nano))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.fix_type))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.flags))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.flags2))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.num_sv))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.lon))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.lat))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.height))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_msl))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.v_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.vel_n))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.vel_e))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.vel_d))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.g_speed))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.head_mot))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.s_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.head_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.p_dop))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.flags3))) as ArrayRef, + Arc::new(FixedSizeListArray::try_new( + Arc::new(Field::new_list_field(DataType::UInt8, false)), + 4, + Arc::new(UInt8Array::from(std::mem::take(&mut self.reserved0))) as ArrayRef, + None, + )?) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.head_veh))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.mag_dec))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.mag_acc))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf21LteConnStatusBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + cell_mcc: Vec, + cell_mnc: Vec, + cell_eci: Vec, + cell_tac: Vec, + earfcn: Vec, + status: Vec, + tech: Vec, + rsrp: Vec, + rsrq: Vec, +} + +impl Tdf21LteConnStatusBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + cell_mcc: Vec::with_capacity(capacity), + cell_mnc: Vec::with_capacity(capacity), + cell_eci: Vec::with_capacity(capacity), + cell_tac: Vec::with_capacity(capacity), + earfcn: Vec::with_capacity(capacity), + status: Vec::with_capacity(capacity), + tech: Vec::with_capacity(capacity), + rsrp: Vec::with_capacity(capacity), + rsrq: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(21).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.cell_mcc.push(cursor.read_u16::()?); + self.cell_mnc.push(cursor.read_u16::()?); + self.cell_eci.push(cursor.read_u32::()?); + self.cell_tac.push(cursor.read_u16::()?); + self.earfcn.push(cursor.read_u32::()?); + self.status.push(cursor.read_u8()?); + self.tech.push(cursor.read_u8()?); + self.rsrp.push(cursor.read_u8()? as f64 / -1.0); + self.rsrq.push(cursor.read_i8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("mcc", DataType::UInt16, false), + Field::new("mnc", DataType::UInt16, false), + Field::new("eci", DataType::UInt32, false), + Field::new("tac", DataType::UInt16, false), + ]), + vec![ + Arc::new(UInt16Array::from(std::mem::take(&mut self.cell_mcc))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.cell_mnc))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.cell_eci))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.cell_tac))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.earfcn))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.status))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.tech))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.rsrq))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf22GlobalstarPktBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + payload: Vec, +} + +impl Tdf22GlobalstarPktBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + payload: Vec::with_capacity(capacity * 9), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(22).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + self.payload.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(FixedSizeListArray::try_new( + Arc::new(Field::new_list_field(DataType::UInt8, false)), + 9, + Arc::new(UInt8Array::from(std::mem::take(&mut self.payload))) as ArrayRef, + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf23AccMagnitudeStdDevBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + count: Vec, + std_dev: Vec, +} + +impl Tdf23AccMagnitudeStdDevBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + count: Vec::with_capacity(capacity), + std_dev: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(23).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.count.push(cursor.read_u32::()?); + self.std_dev.push(cursor.read_u32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.count))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.std_dev))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf24ActivityMetricBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + value: Vec, +} + +impl Tdf24ActivityMetricBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + value: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(24).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.value.push(cursor.read_u32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.value))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf25AlgorithmOutputBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + algorithm_id: Vec, + algorithm_version: Vec, + output: Vec>, +} + +impl Tdf25AlgorithmOutputBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + algorithm_id: Vec::with_capacity(capacity), + algorithm_version: Vec::with_capacity(capacity), + output: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(25).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.algorithm_id.push(cursor.read_u32::()?); + self.algorithm_version + .push(cursor.read_u16::()?); + self.output.push(crate::decoders::tdf_field_read_vla( + cursor, + cursor_start, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.algorithm_id))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take( + &mut self.algorithm_version, + ))) as ArrayRef, + Arc::new(BinaryArray::from_iter_values(std::mem::take( + &mut self.output, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf26RuntimeErrorBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + error_id: Vec, + error_ctx: Vec, +} + +impl Tdf26RuntimeErrorBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + error_id: Vec::with_capacity(capacity), + error_ctx: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(26).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.error_id.push(cursor.read_u32::()?); + self.error_ctx.push(cursor.read_u32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.error_id))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.error_ctx))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf27ChargerEnControlBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + enabled: Vec, +} + +impl Tdf27ChargerEnControlBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + enabled: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(27).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.enabled.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.enabled))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf28GnssFixInfoBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + time_fix: Vec, + location_fix: Vec, + num_sv: Vec, +} + +impl Tdf28GnssFixInfoBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + time_fix: Vec::with_capacity(capacity), + location_fix: Vec::with_capacity(capacity), + num_sv: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(28).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.time_fix.push(cursor.read_u16::()?); + self.location_fix.push(cursor.read_u16::()?); + self.num_sv.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.time_fix))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.location_fix))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.num_sv))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf29BluetoothConnectionBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + address_type: Vec, + address_val: Vec, + connected: Vec, +} + +impl Tdf29BluetoothConnectionBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + address_type: Vec::with_capacity(capacity), + address_val: Vec::with_capacity(capacity), + connected: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(29).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.address_type.push(cursor.read_u8()?); + self.address_val.push(cursor.read_u48::()?); + self.connected.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("type", DataType::UInt8, false), + Field::new("val", DataType::UInt64, false), + ]), + vec![ + Arc::new(UInt8Array::from(std::mem::take(&mut self.address_type))) as ArrayRef, + Arc::new(UInt64Array::from(std::mem::take(&mut self.address_val))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.connected))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf30BluetoothRssiBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + address_type: Vec, + address_val: Vec, + rssi: Vec, +} + +impl Tdf30BluetoothRssiBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + address_type: Vec::with_capacity(capacity), + address_val: Vec::with_capacity(capacity), + rssi: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(30).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.address_type.push(cursor.read_u8()?); + self.address_val.push(cursor.read_u48::()?); + self.rssi.push(cursor.read_i8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("type", DataType::UInt8, false), + Field::new("val", DataType::UInt64, false), + ]), + vec![ + Arc::new(UInt8Array::from(std::mem::take(&mut self.address_type))) as ArrayRef, + Arc::new(UInt64Array::from(std::mem::take(&mut self.address_val))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.rssi))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf31BluetoothDataThroughputBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + address_type: Vec, + address_val: Vec, + throughput: Vec, +} + +impl Tdf31BluetoothDataThroughputBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + address_type: Vec::with_capacity(capacity), + address_val: Vec::with_capacity(capacity), + throughput: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(31).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.address_type.push(cursor.read_u8()?); + self.address_val.push(cursor.read_u48::()?); + self.throughput.push(cursor.read_i32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("type", DataType::UInt8, false), + Field::new("val", DataType::UInt64, false), + ]), + vec![ + Arc::new(UInt8Array::from(std::mem::take(&mut self.address_type))) as ArrayRef, + Arc::new(UInt64Array::from(std::mem::take(&mut self.address_val))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(Int32Array::from(std::mem::take(&mut self.throughput))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf32AlgorithmClassHistogramBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + algorithm_id: Vec, + algorithm_version: Vec, + classes: Vec>, +} + +impl Tdf32AlgorithmClassHistogramBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + algorithm_id: Vec::with_capacity(capacity), + algorithm_version: Vec::with_capacity(capacity), + classes: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(32).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.algorithm_id.push(cursor.read_u32::()?); + self.algorithm_version + .push(cursor.read_u16::()?); + self.classes.push(crate::decoders::tdf_field_read_vla( + cursor, + cursor_start, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.algorithm_id))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take( + &mut self.algorithm_version, + ))) as ArrayRef, + Arc::new(BinaryArray::from_iter_values(std::mem::take( + &mut self.classes, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf33AlgorithmClassTimeSeriesBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + algorithm_id: Vec, + algorithm_version: Vec, + values: Vec>, +} + +impl Tdf33AlgorithmClassTimeSeriesBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + algorithm_id: Vec::with_capacity(capacity), + algorithm_version: Vec::with_capacity(capacity), + values: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(33).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.algorithm_id.push(cursor.read_u32::()?); + self.algorithm_version + .push(cursor.read_u16::()?); + self.values.push(crate::decoders::tdf_field_read_vla( + cursor, + cursor_start, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.algorithm_id))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take( + &mut self.algorithm_version, + ))) as ArrayRef, + Arc::new(BinaryArray::from_iter_values(std::mem::take( + &mut self.values, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf34LteTacCellsBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + cell_mcc: Vec, + cell_mnc: Vec, + cell_eci: Vec, + cell_tac: Vec, + earfcn: Vec, + rsrp: Vec, + rsrq: Vec, + neighbours_offsets: Vec, + neighbours_earfcn: Vec, + neighbours_pci: Vec, + neighbours_time_diff: Vec, + neighbours_rsrp: Vec, + neighbours_rsrq: Vec, +} + +impl Tdf34LteTacCellsBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + cell_mcc: Vec::with_capacity(capacity), + cell_mnc: Vec::with_capacity(capacity), + cell_eci: Vec::with_capacity(capacity), + cell_tac: Vec::with_capacity(capacity), + earfcn: Vec::with_capacity(capacity), + rsrp: Vec::with_capacity(capacity), + rsrq: Vec::with_capacity(capacity), + neighbours_offsets: vec![0], + neighbours_earfcn: Vec::with_capacity(capacity), + neighbours_pci: Vec::with_capacity(capacity), + neighbours_time_diff: Vec::with_capacity(capacity), + neighbours_rsrp: Vec::with_capacity(capacity), + neighbours_rsrq: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(34).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + fn list_value_field(&self, field_index: usize) -> Arc { + let schema = self.schema(); + match schema.field(field_index).data_type() { + DataType::List(field) => field.clone(), + _ => unreachable!("generated list field index is not a list"), + } + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.cell_mcc.push(cursor.read_u16::()?); + self.cell_mnc.push(cursor.read_u16::()?); + self.cell_eci.push(cursor.read_u32::()?); + self.cell_tac.push(cursor.read_u16::()?); + self.earfcn.push(cursor.read_u32::()?); + self.rsrp.push(cursor.read_u8()? as f64 / -1.0); + self.rsrq.push(cursor.read_i8()?); + { + let bytes_remaining = crate::decoders::vla_bytes_remaining(cursor, cursor_start, size)?; + if bytes_remaining % 10 != 0 { + return Err(Error::new( + ErrorKind::InvalidData, + "Variable-length array does not align to element size", + )); + } + let item_count = bytes_remaining / 10; + for _ in 0..item_count { + self.neighbours_earfcn + .push(cursor.read_u32::()?); + self.neighbours_pci.push(cursor.read_u16::()?); + self.neighbours_time_diff + .push(cursor.read_u16::()? as f64 / 1000.0); + self.neighbours_rsrp.push(cursor.read_u8()? as f64 / -1.0); + self.neighbours_rsrq.push(cursor.read_i8()?); + } + self.neighbours_offsets + .push(*self.neighbours_offsets.last().unwrap() + item_count as i32); + } + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("mcc", DataType::UInt16, false), + Field::new("mnc", DataType::UInt16, false), + Field::new("eci", DataType::UInt32, false), + Field::new("tac", DataType::UInt16, false), + ]), + vec![ + Arc::new(UInt16Array::from(std::mem::take(&mut self.cell_mcc))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.cell_mnc))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.cell_eci))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.cell_tac))) as ArrayRef, + ], + None, + )?) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.earfcn))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.rsrq))) as ArrayRef, + { + let offsets = std::mem::replace(&mut self.neighbours_offsets, vec![0]); + Arc::new(ListArray::try_new( + self.list_value_field(6), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("earfcn", DataType::UInt32, false), + Field::new("pci", DataType::UInt16, false), + Field::new("time_diff", DataType::Float64, false), + Field::new("rsrp", DataType::Float64, false), + Field::new("rsrq", DataType::Int8, false), + ]), + vec![ + Arc::new(UInt32Array::from(std::mem::take( + &mut self.neighbours_earfcn, + ))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.neighbours_pci))) + as ArrayRef, + Arc::new(Float64Array::from(std::mem::take( + &mut self.neighbours_time_diff, + ))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take( + &mut self.neighbours_rsrp, + ))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.neighbours_rsrq))) + as ArrayRef, + ], + None, + )?) as ArrayRef, + None, + )?) as ArrayRef + }, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf35WifiApInfoBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + bssid_val: Vec, + channel: Vec, + rsrp: Vec, +} + +impl Tdf35WifiApInfoBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + bssid_val: Vec::with_capacity(capacity), + channel: Vec::with_capacity(capacity), + rsrp: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(35).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.bssid_val.push(cursor.read_u48::()?); + self.channel.push(cursor.read_u8()?); + self.rsrp.push(cursor.read_i8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![Field::new("val", DataType::UInt64, false)]), + vec![Arc::new(UInt64Array::from(std::mem::take(&mut self.bssid_val))) as ArrayRef], + None, + )?) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.channel))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.rsrp))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf36DeviceTiltBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + cosine: Vec, +} + +impl Tdf36DeviceTiltBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + cosine: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(36).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.cosine.push(cursor.read_f32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Float32Array::from(std::mem::take(&mut self.cosine))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf37Nrf9xGnssPvtBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + lat: Vec, + lon: Vec, + height: Vec, + h_acc: Vec, + v_acc: Vec, + h_speed: Vec, + h_speed_acc: Vec, + v_speed: Vec, + v_speed_acc: Vec, + head_mot: Vec, + head_acc: Vec, + year: Vec, + month: Vec, + day: Vec, + hour: Vec, + min: Vec, + sec: Vec, + ms: Vec, + p_dop: Vec, + h_dop: Vec, + v_dop: Vec, + t_dop: Vec, + flags: Vec, + num_sv: Vec, +} + +impl Tdf37Nrf9xGnssPvtBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + lat: Vec::with_capacity(capacity), + lon: Vec::with_capacity(capacity), + height: Vec::with_capacity(capacity), + h_acc: Vec::with_capacity(capacity), + v_acc: Vec::with_capacity(capacity), + h_speed: Vec::with_capacity(capacity), + h_speed_acc: Vec::with_capacity(capacity), + v_speed: Vec::with_capacity(capacity), + v_speed_acc: Vec::with_capacity(capacity), + head_mot: Vec::with_capacity(capacity), + head_acc: Vec::with_capacity(capacity), + year: Vec::with_capacity(capacity), + month: Vec::with_capacity(capacity), + day: Vec::with_capacity(capacity), + hour: Vec::with_capacity(capacity), + min: Vec::with_capacity(capacity), + sec: Vec::with_capacity(capacity), + ms: Vec::with_capacity(capacity), + p_dop: Vec::with_capacity(capacity), + h_dop: Vec::with_capacity(capacity), + v_dop: Vec::with_capacity(capacity), + t_dop: Vec::with_capacity(capacity), + flags: Vec::with_capacity(capacity), + num_sv: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(37).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.lat + .push(cursor.read_i32::()? as f64 / 10000000.0); + self.lon + .push(cursor.read_i32::()? as f64 / 10000000.0); + self.height + .push(cursor.read_i32::()? as f64 / 1000.0); + self.h_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.v_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.h_speed + .push(cursor.read_i32::()? as f64 / 1000.0); + self.h_speed_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.v_speed + .push(cursor.read_i32::()? as f64 / 1000.0); + self.v_speed_acc + .push(cursor.read_u32::()? as f64 / 1000.0); + self.head_mot + .push(cursor.read_i32::()? as f64 / 100000.0); + self.head_acc + .push(cursor.read_u32::()? as f64 / 100000.0); + self.year.push(cursor.read_u16::()?); + self.month.push(cursor.read_u8()?); + self.day.push(cursor.read_u8()?); + self.hour.push(cursor.read_u8()?); + self.min.push(cursor.read_u8()?); + self.sec.push(cursor.read_u8()?); + self.ms.push(cursor.read_u16::()?); + self.p_dop + .push(cursor.read_u16::()? as f64 / 100.0); + self.h_dop + .push(cursor.read_u16::()? as f64 / 100.0); + self.v_dop + .push(cursor.read_u16::()? as f64 / 100.0); + self.t_dop + .push(cursor.read_u16::()? as f64 / 100.0); + self.flags.push(cursor.read_u8()?); + self.num_sv.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.lat))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.lon))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.height))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.v_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_speed))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_speed_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.v_speed))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.v_speed_acc))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.head_mot))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.head_acc))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.year))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.month))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.day))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.hour))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.min))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.sec))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.ms))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.p_dop))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.h_dop))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.v_dop))) as ArrayRef, + Arc::new(Float64Array::from(std::mem::take(&mut self.t_dop))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.flags))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.num_sv))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf38BatteryChargeAccumulatedBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + charge: Vec, +} + +impl Tdf38BatteryChargeAccumulatedBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + charge: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(38).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.charge.push(cursor.read_i32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int32Array::from(std::mem::take(&mut self.charge))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf39InfuseBluetoothRssiBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + infuse_id: Vec, + rssi: Vec, +} + +impl Tdf39InfuseBluetoothRssiBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + infuse_id: Vec::with_capacity(capacity), + rssi: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(39).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.infuse_id.push(cursor.read_u64::()?); + self.rssi.push(cursor.read_i8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt64Array::from(std::mem::take(&mut self.infuse_id))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.rssi))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf40AdcRaw8Builder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + val: Vec, +} + +impl Tdf40AdcRaw8Builder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + val: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(40).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.val.push(cursor.read_i8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.val))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf41AdcRaw16Builder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + val: Vec, +} + +impl Tdf41AdcRaw16Builder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + val: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(41).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.val.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.val))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf42AdcRaw32Builder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + val: Vec, +} + +impl Tdf42AdcRaw32Builder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + val: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(42).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.val.push(cursor.read_i32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int32Array::from(std::mem::take(&mut self.val))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf43AnnotationBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + timestamp: Vec, + event: Vec, +} + +impl Tdf43AnnotationBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + timestamp: Vec::with_capacity(capacity), + event: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(43).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.timestamp.push(cursor.read_u32::()?); + self.event.push(tdf_field_read_string_to_string( + cursor, + cursor_start, + 0, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.timestamp))) as ArrayRef, + Arc::new(StringArray::from_iter_values(std::mem::take( + &mut self.event, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf44LoraRxBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + snr: Vec, + rssi: Vec, + payload: Vec>, +} + +impl Tdf44LoraRxBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + snr: Vec::with_capacity(capacity), + rssi: Vec::with_capacity(capacity), + payload: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(44).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.snr.push(cursor.read_i8()?); + self.rssi.push(cursor.read_i16::()?); + self.payload.push(crate::decoders::tdf_field_read_vla( + cursor, + cursor_start, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.snr))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.rssi))) as ArrayRef, + Arc::new(BinaryArray::from_iter_values(std::mem::take( + &mut self.payload, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf45LoraTxBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + payload: Vec>, +} + +impl Tdf45LoraTxBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + payload: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(45).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.payload.push(crate::decoders::tdf_field_read_vla( + cursor, + cursor_start, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(BinaryArray::from_iter_values(std::mem::take( + &mut self.payload, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf46IdxArrayFreqBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + tdf_id: Vec, + frequency: Vec, +} + +impl Tdf46IdxArrayFreqBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + tdf_id: Vec::with_capacity(capacity), + frequency: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(46).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.tdf_id.push(cursor.read_u16::()?); + self.frequency.push(cursor.read_u32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.tdf_id))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.frequency))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf47IdxArrayPeriodBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + tdf_id: Vec, + period: Vec, +} + +impl Tdf47IdxArrayPeriodBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + tdf_id: Vec::with_capacity(capacity), + period: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(47).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.tdf_id.push(cursor.read_u16::()?); + self.period.push(cursor.read_u32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.tdf_id))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.period))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf48WifiConnectedBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + network_bssid: Vec, + network_band: Vec, + network_channel: Vec, + network_iface_mode: Vec, + network_link_mode: Vec, + network_security: Vec, + network_rssi: Vec, + network_beacon_interval: Vec, + network_twt_capable: Vec, +} + +impl Tdf48WifiConnectedBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + network_bssid: Vec::with_capacity(capacity), + network_band: Vec::with_capacity(capacity), + network_channel: Vec::with_capacity(capacity), + network_iface_mode: Vec::with_capacity(capacity), + network_link_mode: Vec::with_capacity(capacity), + network_security: Vec::with_capacity(capacity), + network_rssi: Vec::with_capacity(capacity), + network_beacon_interval: Vec::with_capacity(capacity), + network_twt_capable: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(48).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.network_bssid.push(cursor.read_u48::()?); + self.network_band.push(cursor.read_u8()?); + self.network_channel.push(cursor.read_u8()?); + self.network_iface_mode.push(cursor.read_u8()?); + self.network_link_mode.push(cursor.read_u8()?); + self.network_security.push(cursor.read_u8()?); + self.network_rssi.push(cursor.read_i8()?); + self.network_beacon_interval + .push(cursor.read_u16::()?); + self.network_twt_capable.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(StructArray::try_new( + Fields::from(vec![ + Field::new("bssid", DataType::UInt64, false), + Field::new("band", DataType::UInt8, false), + Field::new("channel", DataType::UInt8, false), + Field::new("iface_mode", DataType::UInt8, false), + Field::new("link_mode", DataType::UInt8, false), + Field::new("security", DataType::UInt8, false), + Field::new("rssi", DataType::Int8, false), + Field::new("beacon_interval", DataType::UInt16, false), + Field::new("twt_capable", DataType::UInt8, false), + ]), + vec![ + Arc::new(UInt64Array::from(std::mem::take(&mut self.network_bssid))) + as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.network_band))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.network_channel))) + as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take( + &mut self.network_iface_mode, + ))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take( + &mut self.network_link_mode, + ))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.network_security))) + as ArrayRef, + Arc::new(Int8Array::from(std::mem::take(&mut self.network_rssi))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take( + &mut self.network_beacon_interval, + ))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take( + &mut self.network_twt_capable, + ))) as ArrayRef, + ], + None, + )?) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf49WifiConnectionFailedBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + reason: Vec, +} + +impl Tdf49WifiConnectionFailedBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + reason: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(49).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.reason.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.reason))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf50WifiDisconnectedBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + reason: Vec, +} + +impl Tdf50WifiDisconnectedBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + reason: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(50).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.reason.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.reason))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf51NetworkScanCountBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + num_wifi: Vec, + num_lte: Vec, +} + +impl Tdf51NetworkScanCountBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + num_wifi: Vec::with_capacity(capacity), + num_lte: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(51).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.num_wifi.push(cursor.read_u8()?); + self.num_lte.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.num_wifi))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.num_lte))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf52ExceptionStackFrameBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + frame_offsets: Vec, + frame: Vec, +} + +impl Tdf52ExceptionStackFrameBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + frame_offsets: vec![0], + frame: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(52).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + fn list_value_field(&self, field_index: usize) -> Arc { + let schema = self.schema(); + match schema.field(field_index).data_type() { + DataType::List(field) => field.clone(), + _ => unreachable!("generated list field index is not a list"), + } + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + { + let bytes_remaining = crate::decoders::vla_bytes_remaining(cursor, cursor_start, size)?; + if bytes_remaining % 4 != 0 { + return Err(Error::new( + ErrorKind::InvalidData, + "Variable-length array does not align to element size", + )); + } + let item_count = bytes_remaining / 4; + for _ in 0..item_count { + self.frame.push(cursor.read_u32::()?); + } + self.frame_offsets + .push(*self.frame_offsets.last().unwrap() + item_count as i32); + } + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + { + let offsets = std::mem::replace(&mut self.frame_offsets, vec![0]); + Arc::new(ListArray::try_new( + self.list_value_field(2), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(UInt32Array::from(std::mem::take(&mut self.frame))) as ArrayRef, + None, + )?) as ArrayRef + }, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf53BatteryVoltageBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + voltage: Vec, +} + +impl Tdf53BatteryVoltageBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + voltage: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(53).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.voltage.push(cursor.read_u16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.voltage))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf54BatterySocBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + soc: Vec, +} + +impl Tdf54BatterySocBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + soc: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(54).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.soc.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.soc))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf55StateEventSetBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + state: Vec, +} + +impl Tdf55StateEventSetBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + state: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(55).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.state.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.state))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf56StateEventClearedBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + state: Vec, +} + +impl Tdf56StateEventClearedBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + state: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(56).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.state.push(cursor.read_u8()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.state))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf57StateDurationBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + state: Vec, + duration: Vec, +} + +impl Tdf57StateDurationBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + state: Vec::with_capacity(capacity), + duration: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(57).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.state.push(cursor.read_u8()?); + self.duration.push(cursor.read_u32::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt8Array::from(std::mem::take(&mut self.state))) as ArrayRef, + Arc::new(UInt32Array::from(std::mem::take(&mut self.duration))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf58Pcm16bitChanLeftBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + val: Vec, +} + +impl Tdf58Pcm16bitChanLeftBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + val: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(58).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.val.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.val))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf59Pcm16bitChanRightBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + val: Vec, +} + +impl Tdf59Pcm16bitChanRightBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + val: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(59).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.val.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.val))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf60Pcm16bitChanDualBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + left: Vec, + right: Vec, +} + +impl Tdf60Pcm16bitChanDualBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + left: Vec::with_capacity(capacity), + right: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(60).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.left.push(cursor.read_i16::()?); + self.right.push(cursor.read_i16::()?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.left))) as ArrayRef, + Arc::new(Int16Array::from(std::mem::take(&mut self.right))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} + +pub struct Tdf61KvsValueChangedBuilder { + row_timestamp: Vec>, + row_sample_idx: Vec>, + key: Vec, + value: Vec>, +} + +impl Tdf61KvsValueChangedBuilder { + pub fn new(capacity: usize) -> Self { + Self { + row_timestamp: Vec::with_capacity(capacity), + row_sample_idx: Vec::with_capacity(capacity), + key: Vec::with_capacity(capacity), + value: Vec::with_capacity(capacity), + } + } + + pub fn schema(&self) -> SchemaRef { + tdf_parquet_schema(61).unwrap() + } + + pub fn rows(&self) -> usize { + self.row_timestamp.len() + } + + pub fn append( + &mut self, + meta: TdfParquetRowMeta, + size: u8, + cursor: &mut Cursor<&[u8]>, + ) -> Result<()> { + let cursor_start = cursor.position(); + + self.row_timestamp.push(meta.time_unix_micros); + self.row_sample_idx.push(meta.sample_idx); + self.key.push(cursor.read_u16::()?); + self.value.push(crate::decoders::tdf_field_read_vla( + cursor, + cursor_start, + size, + )?); + + finish_tdf_read(cursor, cursor_start, size) + } + + pub fn finish_batch(&mut self) -> std::result::Result { + let schema = self.schema(); + let columns = vec![ + Arc::new( + TimestampMicrosecondArray::from(std::mem::take(&mut self.row_timestamp)) + .with_timezone("+00:00"), + ) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.row_sample_idx))) as ArrayRef, + Arc::new(UInt16Array::from(std::mem::take(&mut self.key))) as ArrayRef, + Arc::new(BinaryArray::from_iter_values(std::mem::take( + &mut self.value, + ))) as ArrayRef, + ]; + + RecordBatch::try_new(schema, columns) + } +} diff --git a/tdf/src/lib.rs b/tdf/src/lib.rs index 66775fe..f78f7bd 100644 --- a/tdf/src/lib.rs +++ b/tdf/src/lib.rs @@ -1,10 +1,14 @@ use bytemuck; use byteorder::{LittleEndian, ReadBytesExt}; use num::{cast::AsPrimitive, traits::WrappingAdd}; -use std::io::{Cursor, ErrorKind, Read}; +use std::{ + io::{Cursor, ErrorKind, Read}, + path::PathBuf, +}; pub mod decoders; pub mod decoders_csv; +pub mod decoders_parquet; pub mod time; const TDF_TIME_MASK: u16 = 0xC000; @@ -44,6 +48,8 @@ pub trait TdfOutput { fn iter_written(&self) -> impl Iterator, u16), &usize)>; /// Get the number of times a specific TDF was written fn written(&self, remote_id: Option, tdf_id: u16) -> usize; + /// Get the path for the file associated with a specific TDF + fn output_path(self: &Self, remote_id: Option, tdf_id: u16) -> Option; } fn diff_data_reconstruct< diff --git a/tdf/src/time.rs b/tdf/src/time.rs index 7fbfc48..cb4dde7 100644 --- a/tdf/src/time.rs +++ b/tdf/src/time.rs @@ -11,6 +11,11 @@ pub fn tdf_time_to_unix(tdf_time: i64) -> (i64, u32) { (unix_seconds, unix_nano as u32) } +pub fn tdf_time_to_unix_micros(tdf_time: i64) -> i64 { + let (unix_seconds, unix_nano) = tdf_time_to_unix(tdf_time); + (unix_seconds * 1_000_000) + (unix_nano as i64 / 1_000) +} + pub fn tdf_time_to_datetime(tdf_time: i64) -> Option> { let (unix_seconds, unix_nano) = tdf_time_to_unix(tdf_time); DateTime::from_timestamp(unix_seconds as i64, unix_nano as u32)