From 3fa9a2b0ad93a732c5d353d7f9c10be66b3122e2 Mon Sep 17 00:00:00 2001 From: "Clelia (Astra) Bertelli" Date: Mon, 6 Apr 2026 17:16:08 +0200 Subject: [PATCH 1/3] wip: add support for nan --- src/lib.rs | 141 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 117 insertions(+), 24 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3e7c43c..5b69785 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,11 @@ #![deny(clippy::all)] -use std::{collections::HashMap, fmt, fs::File, io::BufWriter}; +use std::{ + collections::{HashMap, HashSet}, + fmt, + fs::File, + io::BufWriter, +}; use anyhow::{anyhow, Result}; @@ -12,10 +17,10 @@ use napi_derive::napi; #[napi] #[derive(Clone)] pub enum ColumnData { - String(Vec), - Integer(Vec), - Float(Vec), - Boolean(Vec), + String(Vec>), + Integer(Vec>), + Float(Vec>), + Boolean(Vec>), } impl ColumnData { @@ -53,7 +58,7 @@ impl fmt::Display for DataType { // ---------------------------------------- HELPER FUNCTIONS FOR DATA MODELS ---------------------------------------- #[napi] -pub fn as_float_array(column: ColumnData) -> Option> { +pub fn as_float_array(column: ColumnData) -> Option>> { match column { ColumnData::Float(f) => Some(f), _ => None, @@ -61,7 +66,7 @@ pub fn as_float_array(column: ColumnData) -> Option> { } #[napi] -pub fn as_int_array(column: ColumnData) -> Option> { +pub fn as_int_array(column: ColumnData) -> Option>> { match column { ColumnData::Integer(f) => Some(f), _ => None, @@ -69,7 +74,7 @@ pub fn as_int_array(column: ColumnData) -> Option> { } #[napi] -pub fn as_boolean_array(column: ColumnData) -> Option> { +pub fn as_boolean_array(column: ColumnData) -> Option>> { match column { ColumnData::Boolean(f) => Some(f), _ => None, @@ -77,7 +82,7 @@ pub fn as_boolean_array(column: ColumnData) -> Option> { } #[napi] -pub fn as_string_array(column: ColumnData) -> Option> { +pub fn as_string_array(column: ColumnData) -> Option>> { match column { ColumnData::String(f) => Some(f), _ => None, @@ -85,22 +90,22 @@ pub fn as_string_array(column: ColumnData) -> Option> { } #[napi] -pub fn to_string_column(data: Vec) -> ColumnData { +pub fn to_string_column(data: Vec>) -> ColumnData { ColumnData::String(data) } #[napi] -pub fn to_float_column(data: Vec) -> ColumnData { +pub fn to_float_column(data: Vec>) -> ColumnData { ColumnData::Float(data) } #[napi] -pub fn to_int_column(data: Vec) -> ColumnData { +pub fn to_int_column(data: Vec>) -> ColumnData { ColumnData::Integer(data) } #[napi] -pub fn to_bool_column(data: Vec) -> ColumnData { +pub fn to_bool_column(data: Vec>) -> ColumnData { ColumnData::Boolean(data) } @@ -163,6 +168,82 @@ impl DataFrame { self.columns.clone() } + #[napi] + pub fn dropna(&mut self) { + let mut idxs = HashSet::new(); + for i in 0..self.len as usize { + let mut has_none = false; + for (_, col) in &self.columns { + match col { + ColumnData::Boolean(b) => { + if b[i].is_none() { + has_none = true; + } + } + ColumnData::Float(b) => { + if b[i].is_none() { + has_none = true; + } + } + ColumnData::String(b) => { + if b[i].is_none() { + has_none = true; + } + } + ColumnData::Integer(b) => { + if b[i].is_none() { + has_none = true; + } + } + } + if has_none { + break; + } + } + if has_none { + idxs.insert(i); + } + } + if !idxs.is_empty() { + for (_, col) in &mut self.columns { + match col { + ColumnData::Boolean(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Float(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Integer(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::String(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + } + } + } + } + #[napi] pub fn write_csv(&self, path: String) -> Result<()> { let file = File::create(&path)?; @@ -178,7 +259,7 @@ impl DataFrame { let col = &self.columns[*key]; match col { ColumnData::Boolean(b) => { - let v = if b[i] { + let v = if b[i].unwrap_or_default() { "true".to_string() } else { "false".to_string() @@ -186,15 +267,15 @@ impl DataFrame { row.push(v); } ColumnData::Float(f) => { - let v = float_buf.format(f[i]).to_string(); + let v = float_buf.format(f[i].unwrap_or(f64::NAN)).to_string(); row.push(v); } ColumnData::Integer(j) => { - let v = int_buf.format(j[i]).to_string(); + let v = int_buf.format(j[i].unwrap_or_default()).to_string(); row.push(v); } ColumnData::String(s) => { - row.push(s[i].clone()); + row.push(s[i].clone().unwrap_or_default()); } } } @@ -265,10 +346,14 @@ pub fn read_csv(path: String) -> Result { let vc = &col_to_vec[*c]; let data: ColumnData = match d { DataType::Boolean => { - let mut v: Vec = vec![]; + let mut v: Vec> = vec![]; for el in vc { + if el.is_empty() { + v.push(None); + continue; + } match str_to_bool(el) { - Some(val) => v.push(val), + Some(val) => v.push(Some(val)), None => { return Err(anyhow!( "Expecting bool at line {:?} for col {}", @@ -281,10 +366,14 @@ pub fn read_csv(path: String) -> Result { ColumnData::Boolean(v) } DataType::Float => { - let mut v: Vec = vec![]; + let mut v: Vec> = vec![]; for s in vc { + if s.is_empty() { + v.push(None); + continue; + } match s.parse::() { - Ok(val) => v.push(val), + Ok(val) => v.push(Some(val)), Err(_) => { return Err(anyhow!( "Expecting float at line {:?} for col {}", @@ -298,14 +387,18 @@ pub fn read_csv(path: String) -> Result { ColumnData::Float(v) } DataType::String => { - let v = vc.iter().map(|s| s.to_string()).collect(); + let v = vc.iter().map(|s| Some(s.to_string())).collect(); ColumnData::String(v) } DataType::Integer => { - let mut v: Vec = vec![]; + let mut v: Vec> = vec![]; for s in vc { + if s.is_empty() { + v.push(None); + continue; + } match s.parse::() { - Ok(val) => v.push(val), + Ok(val) => v.push(Some(val)), Err(_) => { return Err(anyhow!( "Expecting integer at line {:?} for col {}", From ba91da7904190452562582c450b81b56ec08b265 Mon Sep 17 00:00:00 2001 From: "Clelia (Astra) Bertelli" Date: Tue, 7 Apr 2026 11:37:31 +0200 Subject: [PATCH 2/3] feat: tests, renaming --- .gitignore | 1 + __test__/index.spec.ts | 55 ++++++++++ index.d.ts | 26 ++--- index.js | 104 +++++++++---------- src/lib.rs | 60 +++++++++-- testfiles/generated-100.csv | 200 ++++++++++++++++++------------------ testfiles/with-nans.csv | 6 ++ 7 files changed, 280 insertions(+), 172 deletions(-) create mode 100644 testfiles/with-nans.csv diff --git a/.gitignore b/.gitignore index 0ec9a24..687533a 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,4 @@ Cargo.lock testfiles/*.csv !testfiles/generated-100.csv !testfiles/err-*-col*.csv +!testfiles/with-nans.csv diff --git a/__test__/index.spec.ts b/__test__/index.spec.ts index 3f62df5..46c992f 100644 --- a/__test__/index.spec.ts +++ b/__test__/index.spec.ts @@ -189,3 +189,58 @@ test('Write to CSV works correctly', (t) => { } unlinkSync('testfiles/write-test.csv') }) + +test('Dropping null values works correctly', (t) => { + const col1 = toStringColumn(['hello', 'world', null, 'bye', 'moon', 'hey']) + const col2 = toFloatColumn([1.2, 2.3, 0.3, null, 0.5, 8.9]) + const col3 = toIntColumn([4, 5, 6, 7, null, 8]) + const col4 = toBoolColumn([true, false, true, false, true, null]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + col3: col3, + col4: col4, + }) + t.is(df.len, 6) + df.dropNull() + t.is(df.len, 2) + t.deepEqual(asStringArray(df.get('col1')!)!, ['hello', 'world']) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3]) + t.deepEqual(asIntArray(df.get('col3')!)!, [4, 5]) + t.deepEqual(asBooleanArray(df.get('col4')!)!, [true, false]) +}) + +test('Reading null values works correctly', (t) => { + const df = readCsv('testfiles/with-nans.csv') + t.is(df.len, 5) + const col1 = asStringArray(df.get('col1')!)! + t.is(col1.filter((x) => x === null).length, 1) + t.deepEqual(col1, ['hello', null, 'bye', 'test', 'ciao']) + const col2 = asIntArray(df.get('col2')!)! + t.is(col2.filter((x) => x === null).length, 1) + t.deepEqual(col2, [1, 2, null, 3, 4]) + const col3 = asFloatArray(df.get('col3')!)! + t.is(col3.filter((x) => x === null).length, 1) + t.deepEqual(col3, [0.1, 0.2, 0.3, null, 0.4]) + const col4 = asBooleanArray(df.get('col4')!)! + t.is(col4.filter((x) => x === null).length, 1) + t.deepEqual(col4, [true, false, true, false, null]) +}) + +test('Filling null values works correctly', (t) => { + const df = readCsv('testfiles/with-nans.csv') + df.fillNull() + t.is(df.len, 5) + const col1 = asStringArray(df.get('col1')!)! + t.is(col1.filter((x) => x === null).length, 0) + t.deepEqual(col1, ['hello', '', 'bye', 'test', 'ciao']) + const col2 = asIntArray(df.get('col2')!)! + t.is(col2.filter((x) => x === null).length, 0) + t.deepEqual(col2, [1, 2, 0, 3, 4]) + const col3 = asFloatArray(df.get('col3')!)! + t.is(col3.filter((x) => x === null).length, 0) + t.deepEqual(col3, [0.1, 0.2, 0.3, 0.0, 0.4]) + const col4 = asBooleanArray(df.get('col4')!)! + t.is(col4.filter((x) => x === null).length, 0) + t.deepEqual(col4, [true, false, true, false, false]) +}) diff --git a/index.d.ts b/index.d.ts index 25b9470..3d259aa 100644 --- a/index.d.ts +++ b/index.d.ts @@ -7,22 +7,24 @@ export declare class DataFrame { colDtype(col: string): DataType | null get(col: string): ColumnData | null get columns(): Record + dropNull(): void + fillNull(): void writeCsv(path: string): void } -export declare function asBooleanArray(column: ColumnData): Array | null +export declare function asBooleanArray(column: ColumnData): Array | null -export declare function asFloatArray(column: ColumnData): Array | null +export declare function asFloatArray(column: ColumnData): Array | null -export declare function asIntArray(column: ColumnData): Array | null +export declare function asIntArray(column: ColumnData): Array | null -export declare function asStringArray(column: ColumnData): Array | null +export declare function asStringArray(column: ColumnData): Array | null export type ColumnData = - | { type: 'String'; field0: Array } - | { type: 'Integer'; field0: Array } - | { type: 'Float'; field0: Array } - | { type: 'Boolean'; field0: Array } + | { type: 'String'; field0: Array } + | { type: 'Integer'; field0: Array } + | { type: 'Float'; field0: Array } + | { type: 'Boolean'; field0: Array } export declare const enum DataType { String = 0, @@ -33,10 +35,10 @@ export declare const enum DataType { export declare function readCsv(path: string): DataFrame -export declare function toBoolColumn(data: Array): ColumnData +export declare function toBoolColumn(data: Array): ColumnData -export declare function toFloatColumn(data: Array): ColumnData +export declare function toFloatColumn(data: Array): ColumnData -export declare function toIntColumn(data: Array): ColumnData +export declare function toIntColumn(data: Array): ColumnData -export declare function toStringColumn(data: Array): ColumnData +export declare function toStringColumn(data: Array): ColumnData diff --git a/index.js b/index.js index c53cb9c..42b8acf 100644 --- a/index.js +++ b/index.js @@ -77,8 +77,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-android-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-android-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -93,8 +93,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-android-arm-eabi') const bindingPackageVersion = require('@cle-does-things/sunbears-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -114,8 +114,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-x64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-x64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -130,8 +130,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-x64-msvc') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -147,8 +147,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-ia32-msvc') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -163,8 +163,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-arm64-msvc') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -182,8 +182,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-darwin-universal') const bindingPackageVersion = require('@cle-does-things/sunbears-darwin-universal/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -198,8 +198,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-darwin-x64') const bindingPackageVersion = require('@cle-does-things/sunbears-darwin-x64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -214,8 +214,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-darwin-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-darwin-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -234,8 +234,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-freebsd-x64') const bindingPackageVersion = require('@cle-does-things/sunbears-freebsd-x64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -250,8 +250,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-freebsd-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -271,8 +271,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-x64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -287,8 +287,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-x64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -305,8 +305,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -321,8 +321,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -339,8 +339,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm-musleabihf') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -355,8 +355,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm-gnueabihf') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -373,8 +373,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-loong64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-loong64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -389,8 +389,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-loong64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-loong64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -407,8 +407,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-riscv64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -423,8 +423,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-riscv64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -440,8 +440,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-ppc64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -456,8 +456,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-s390x-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -476,8 +476,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-openharmony-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -492,8 +492,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-openharmony-x64') const bindingPackageVersion = require('@cle-does-things/sunbears-openharmony-x64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -508,8 +508,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-openharmony-arm') const bindingPackageVersion = require('@cle-does-things/sunbears-openharmony-arm/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { diff --git a/src/lib.rs b/src/lib.rs index 5b69785..0ed36b0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -169,11 +169,11 @@ impl DataFrame { } #[napi] - pub fn dropna(&mut self) { + pub fn drop_null(&mut self) { let mut idxs = HashSet::new(); for i in 0..self.len as usize { let mut has_none = false; - for (_, col) in &self.columns { + for col in self.columns.values() { match col { ColumnData::Boolean(b) => { if b[i].is_none() { @@ -205,7 +205,7 @@ impl DataFrame { } } if !idxs.is_empty() { - for (_, col) in &mut self.columns { + for col in self.columns.values_mut() { match col { ColumnData::Boolean(b) => { let mut i = 0; @@ -241,6 +241,43 @@ impl DataFrame { } } } + self.len -= idxs.len() as u32; + } + } + + #[napi] + pub fn fill_null(&mut self) { + for col in self.columns.values_mut() { + match col { + ColumnData::Boolean(b) => { + for item in b.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(false); + } + } + } + ColumnData::Float(f) => { + for item in f.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(0_f64); + } + } + } + ColumnData::String(s) => { + for item in s.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(String::new()); + } + } + } + ColumnData::Integer(j) => { + for item in j.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(0_i64); + } + } + } + } } } @@ -267,7 +304,7 @@ impl DataFrame { row.push(v); } ColumnData::Float(f) => { - let v = float_buf.format(f[i].unwrap_or(f64::NAN)).to_string(); + let v = float_buf.format(f[i].unwrap_or_default()).to_string(); row.push(v); } ColumnData::Integer(j) => { @@ -346,7 +383,7 @@ pub fn read_csv(path: String) -> Result { let vc = &col_to_vec[*c]; let data: ColumnData = match d { DataType::Boolean => { - let mut v: Vec> = vec![]; + let mut v: Vec> = Vec::with_capacity(vc.len()); for el in vc { if el.is_empty() { v.push(None); @@ -366,7 +403,7 @@ pub fn read_csv(path: String) -> Result { ColumnData::Boolean(v) } DataType::Float => { - let mut v: Vec> = vec![]; + let mut v: Vec> = Vec::with_capacity(vc.len()); for s in vc { if s.is_empty() { v.push(None); @@ -387,11 +424,18 @@ pub fn read_csv(path: String) -> Result { ColumnData::Float(v) } DataType::String => { - let v = vc.iter().map(|s| Some(s.to_string())).collect(); + let mut v: Vec> = Vec::with_capacity(vc.len()); + for s in vc { + if s.is_empty() { + v.push(None); + continue; + } + v.push(Some(s.to_string())); + } ColumnData::String(v) } DataType::Integer => { - let mut v: Vec> = vec![]; + let mut v: Vec> = Vec::with_capacity(vc.len()); for s in vc { if s.is_empty() { v.push(None); diff --git a/testfiles/generated-100.csv b/testfiles/generated-100.csv index ea61a40..ed14cfb 100644 --- a/testfiles/generated-100.csv +++ b/testfiles/generated-100.csv @@ -1,101 +1,101 @@ name,age,is_working,height,phone_number,years_of_experience -Anne,24,false,172.06195385839587,+1 1234,3.767044082274923 -Enid,36,true,164.17546867223956,+1 1234,3.9987730641281765 -Francis,45,true,167.66954681406762,0043-1234,0.4383413027655949 -Enid,34,true,161.30458950930952,+1 1234,1.0983771829465512 -Charles,25,false,151.67913999785412,0043-1234,2.434259647739991 -Dylan,31,true,161.16597974594205,0043-1234,3.276539501458718 -Charles,43,false,153.78110220498175,+1 1234,4.134995277849777 -Charles,25,true,166.40188604691568,0043-1234,3.4263596384326083 -Bob,23,false,158.38593329361092,0043-1234,1.0320864141187804 -Bob,22,false,174.30421224335603,0043-1234,0.30139282444920124 -Charles,25,false,167.36986172098074,+1 1234,3.6070863321158333 -Bob,47,true,155.77584934965734,+1 1234,0.3358753495974762 -Anne,47,true,152.07245609048525,0043-1234,3.978476753629136 -Anne,35,false,163.8284506948372,0043-1234,4.7375086847218375 -Anne,49,true,180.1237482982292,+1 1234,1.4734837130167757 -Francis,31,false,171.39476207206496,+1 1234,2.057482207163191 -Charles,20,false,150.85891722659494,+1 1234,1.8076413190188547 -Charles,26,true,157.73556835468887,0043-1234,3.2390834201030048 -Bob,28,false,180.22280208552263,0043-1234,0.11382435925677648 -Francis,50,true,162.39526970619806,+1 1234,0.8249470790353828 -Charles,39,true,154.01212102360162,+1 1234,1.6846034197031907 -Francis,27,true,173.5486689216849,+1 1234,3.9985312398191337 -Francis,23,false,177.5849194905182,0043-1234,2.6625711310679345 -Francis,20,true,176.23107677217902,+1 1234,2.36102935780334 -Francis,34,true,180.7011235918709,0043-1234,1.482859764730454 -Anne,28,true,172.2611271479767,+1 1234,0.8026755462524306 -Francis,35,true,163.02339898361976,+1 1234,0.13368333645175268 -Dylan,21,false,164.56067064798725,+1 1234,1.1321786041675934 -Anne,41,true,179.442405598062,0043-1234,2.4910230451061053 -Dylan,40,true,169.0704375228679,+1 1234,3.767070661054649 -Bob,45,false,158.52554354403676,0043-1234,3.389188917679733 -Anne,49,true,178.3749017440682,+1 1234,1.2717571652842483 -Francis,42,false,179.89635183986815,+1 1234,0.7228721084532994 -Francis,50,false,162.16345519279804,0043-1234,2.1467121727357386 -Anne,25,false,151.85714506267925,0043-1234,2.0106703515331454 -Enid,43,true,154.44252150760803,0043-1234,0.10012231117163695 -Francis,33,false,169.75224887442954,0043-1234,2.1155764577654237 -Dylan,21,true,163.42614142253774,+1 1234,4.1636259380519896 -Charles,39,false,177.19248309034617,0043-1234,0.7043319312506218 -Dylan,27,false,163.78268872264073,0043-1234,0.7933675094289101 -Enid,22,true,151.69475029878475,0043-1234,3.427832466709869 -Anne,24,true,167.0624790612009,0043-1234,3.208757157964802 -Enid,22,true,161.48780558514278,+1 1234,1.8090922295976357 -Francis,49,true,161.18349998788094,+1 1234,1.0264327081269466 -Enid,30,true,155.8820687964176,0043-1234,1.0565103129209166 -Anne,50,false,163.58681127771385,0043-1234,2.834026981707148 -Enid,25,true,165.3165808966044,0043-1234,2.907435266125858 -Dylan,33,true,178.4105287916339,0043-1234,3.1364447532695845 -Francis,24,false,153.3751836951103,0043-1234,3.573562444922654 -Francis,33,true,175.3551671500021,0043-1234,0.8777011127810928 -Bob,28,false,167.79756345857308,0043-1234,2.5037076919560795 -Dylan,31,true,151.22287551814932,+1 1234,3.909823087240767 -Anne,24,true,159.32157999459298,0043-1234,3.304525677644712 -Dylan,45,false,179.506794367215,0043-1234,1.94228733904784 -Enid,47,false,153.09902751239227,0043-1234,1.2257610300558768 -Enid,24,false,155.81295425867603,0043-1234,2.5198447985382337 -Dylan,35,true,177.8206091922323,0043-1234,3.11915384637018 -Charles,36,false,169.68986298923735,+1 1234,2.040447345083472 -Bob,26,true,175.03218264042204,+1 1234,0.969361969266481 -Bob,33,false,172.40702314693513,0043-1234,3.0529996726430175 -Dylan,21,true,180.24598488538015,+1 1234,2.4544469347421005 -Charles,47,false,152.5271909335372,+1 1234,4.862982108147397 -Bob,31,true,174.82341433179184,+1 1234,4.772525888375782 -Dylan,47,false,159.68297409447837,0043-1234,2.910096160363199 -Dylan,26,false,163.5897866306247,0043-1234,4.5364911850626335 -Dylan,23,true,150.732788122647,0043-1234,4.315366231031446 -Enid,23,false,171.53990655114706,+1 1234,0.5693450705816476 -Enid,48,false,163.53419005719897,0043-1234,1.2921866236585322 -Francis,47,false,151.76618196211368,+1 1234,2.3720314158733604 -Charles,27,true,159.52354929775623,+1 1234,4.867960488247686 -Enid,49,false,162.074604287669,0043-1234,3.7767032840851327 -Bob,40,true,180.6973555225904,+1 1234,4.976864922452478 -Dylan,43,true,173.8940011972604,+1 1234,3.840024105102519 -Enid,35,false,170.53003190168505,0043-1234,3.146912731530201 -Charles,45,false,175.9191299427927,0043-1234,0.13449729759980267 -Francis,37,false,178.81323593962227,+1 1234,1.8046065918879781 -Francis,37,true,171.64751648501152,0043-1234,1.2121177903237317 -Dylan,21,true,174.23500345515865,0043-1234,2.40193633389855 -Enid,23,true,166.64177130219286,0043-1234,1.3485286041527245 -Enid,28,true,167.84237767602943,+1 1234,3.5604680454430335 -Anne,47,false,173.3829667919652,0043-1234,3.4355520047255492 -Charles,42,true,172.32056905611879,0043-1234,4.572668974525715 -Francis,26,true,153.87122280420334,0043-1234,1.1098493391586837 -Bob,38,false,154.80156174298156,+1 1234,1.09022613940166 -Charles,30,true,175.78649302190107,+1 1234,2.8155094550286215 -Anne,24,false,173.6683115337848,0043-1234,3.411235438552893 -Anne,47,true,164.74234862130373,+1 1234,4.395743541956674 -Charles,39,true,160.344588323495,0043-1234,4.617831663085374 -Anne,41,false,154.21935376553893,0043-1234,3.8765168318277285 -Enid,38,true,161.36130856960816,0043-1234,4.292390362269066 -Dylan,22,true,151.97403458363888,0043-1234,1.511052542912881 -Charles,30,true,160.8150715590866,+1 1234,3.4227094005312617 -Francis,36,false,153.92000405422075,0043-1234,2.57570905818981 -Francis,49,true,162.03322009490762,+1 1234,1.3991670356908592 -Enid,39,false,180.7453190324777,0043-1234,4.170054416280345 -Dylan,30,true,157.385569900398,0043-1234,4.456103378715245 -Enid,24,false,151.93522138656664,0043-1234,3.456877765620685 -Charles,31,false,178.33999739259212,+1 1234,4.708568405839408 -Anne,50,false,151.1441208606709,0043-1234,3.0190218516256966 -Anne,20,false,180.56604162487267,+1 1234,2.47731278312586 +Dylan,42,false,152.2337643232597,0043-1234,2.710603836108621 +Enid,29,true,169.44529918227406,+1 1234,1.9186595310390577 +Dylan,26,true,155.06276698093373,+1 1234,4.300156885007877 +Anne,21,false,154.98925970803663,0043-1234,1.8933188282167404 +Anne,44,true,180.21502717549743,0043-1234,3.603969658623612 +Bob,40,true,165.40415901447975,+1 1234,2.845444565703957 +Francis,33,false,174.44508280771316,0043-1234,2.9959578228789834 +Dylan,22,false,173.76493458418864,0043-1234,4.829538663354264 +Anne,20,true,178.8647504968728,+1 1234,2.1807362808824395 +Charles,24,false,164.36863959493428,0043-1234,3.5572587149384427 +Enid,30,false,165.6636593782633,+1 1234,3.2930585952088283 +Enid,30,true,171.84380730396535,0043-1234,0.29869230763984844 +Dylan,36,true,155.16495731529344,+1 1234,1.8442913559545064 +Bob,49,false,156.72531783682777,+1 1234,3.9517646609102455 +Anne,28,false,175.16416650994768,+1 1234,0.5932506406862179 +Francis,50,true,167.83948718687446,0043-1234,4.858709505152615 +Bob,32,true,176.05658608461243,0043-1234,3.8678770628403956 +Charles,28,false,157.65302556188283,+1 1234,3.85713775381506 +Francis,31,true,157.05159155275442,0043-1234,3.7941829176287065 +Bob,37,true,157.8242223834105,0043-1234,3.1102010145933776 +Dylan,41,true,152.31187822580569,+1 1234,3.5510443626328847 +Anne,31,false,164.63707496075145,+1 1234,1.8954734566400697 +Charles,49,false,159.6152622470992,+1 1234,1.4710266707931205 +Bob,30,false,173.04776580077328,0043-1234,1.7684680457734039 +Enid,26,true,161.16426180165936,0043-1234,2.5947287988698275 +Bob,33,true,164.584792448029,0043-1234,0.4170630327377409 +Anne,46,true,156.8716989148629,0043-1234,1.9682008452336965 +Charles,28,false,170.92400270139427,+1 1234,3.694392339828717 +Charles,38,false,160.02548944220067,0043-1234,3.7864176354739367 +Enid,38,true,169.11415760888443,+1 1234,2.553769773167301 +Dylan,40,true,156.70256919306888,+1 1234,3.2569541662817443 +Dylan,45,true,178.79902539954585,0043-1234,3.320434569767923 +Dylan,26,true,170.0472625467359,0043-1234,2.3723283322979123 +Francis,49,true,162.2375687558636,+1 1234,4.749812668737068 +Dylan,23,true,156.6866835960943,0043-1234,1.0127174368942098 +Bob,38,true,180.3465341266976,+1 1234,0.35644809979627334 +Enid,36,false,177.68158834314482,+1 1234,1.8603889825422584 +Anne,33,true,180.34367433810976,+1 1234,2.859833382035033 +Charles,49,false,159.06132113458898,+1 1234,2.356082572340613 +Charles,27,false,176.79719018961643,0043-1234,2.361085336622755 +Charles,32,true,165.5038587798262,+1 1234,4.701131180253473 +Bob,50,true,156.2732796282787,+1 1234,2.9871409774632256 +Enid,34,true,168.4792444229309,0043-1234,3.167110474391576 +Bob,41,true,170.65682514988467,+1 1234,3.1842293644017765 +Charles,30,true,180.11440728933985,+1 1234,3.3340682130335937 +Anne,24,true,168.82752873187513,+1 1234,4.074559876799402 +Enid,50,true,173.80312298394057,+1 1234,2.603607644773761 +Dylan,43,true,171.2498164542436,+1 1234,2.995157503942086 +Bob,29,true,176.5332297667723,0043-1234,4.792132532376961 +Enid,24,false,156.3152333573472,+1 1234,2.279840425113064 +Dylan,33,true,164.41672242642258,0043-1234,2.046726377844414 +Charles,30,true,152.98655908018037,+1 1234,3.225492395302485 +Charles,29,true,158.15269781789218,+1 1234,2.232872472543561 +Francis,30,false,170.5090759393567,0043-1234,4.826935025835683 +Francis,34,false,163.93109492473243,0043-1234,1.6240047886100537 +Francis,29,true,172.700773237117,+1 1234,1.3127190206762955 +Dylan,26,true,162.67195903287686,0043-1234,0.33222993830582315 +Charles,47,true,158.14905659054304,+1 1234,1.013882418974281 +Charles,44,true,166.62412538910291,+1 1234,0.045743111899582134 +Anne,28,true,166.95355019282664,+1 1234,0.7982170079180548 +Francis,42,false,156.79758594634148,+1 1234,1.3617907272217888 +Francis,49,true,172.42424362645298,+1 1234,4.342859717299524 +Francis,39,false,161.55673026150154,+1 1234,4.289002272448447 +Dylan,38,true,156.78262933687031,+1 1234,3.2037219868478406 +Charles,31,false,168.94251163149627,0043-1234,2.966612991432437 +Francis,30,false,168.53091362629632,0043-1234,0.27391352355643117 +Dylan,39,true,169.9544991081737,+1 1234,1.0506048994344854 +Francis,29,true,163.69812373179843,0043-1234,4.0952079762730875 +Francis,49,true,163.8229577296399,+1 1234,2.907230653232779 +Enid,38,false,155.63267386685342,0043-1234,2.6336821984845926 +Charles,21,true,171.32125410820913,+1 1234,3.2726028187887195 +Anne,20,false,169.95160147856606,0043-1234,3.636289344725836 +Bob,36,false,171.25000202065186,0043-1234,4.996409278759291 +Bob,20,true,153.68559944681937,0043-1234,4.479337308398978 +Charles,27,false,171.0377797995193,0043-1234,3.3357404123674828 +Enid,36,true,160.66327145682146,0043-1234,2.608196831381723 +Anne,25,false,150.86609499189495,0043-1234,4.523359997234933 +Charles,27,false,161.79640810379877,+1 1234,3.796289654423112 +Francis,46,true,161.16991583857177,0043-1234,2.3019539257627075 +Anne,22,false,177.88531794342396,+1 1234,2.6711824210696182 +Anne,46,true,170.02573691414202,0043-1234,3.3105244025305636 +Charles,30,false,177.56599570674095,0043-1234,0.36264731267778494 +Anne,22,false,174.98008638974463,0043-1234,3.981181990095814 +Francis,38,true,178.3637457193352,+1 1234,3.0846262408736 +Dylan,21,true,159.1454642739654,0043-1234,3.6381043367705663 +Charles,38,false,164.15859475199443,+1 1234,3.0465150075483445 +Anne,21,false,158.56246926321796,+1 1234,2.4885388005852915 +Charles,20,true,151.5147102387871,0043-1234,1.259301200465609 +Enid,47,false,175.338097479699,0043-1234,1.8438560301641096 +Bob,34,false,179.9105013310785,0043-1234,2.0529343125484867 +Francis,21,true,163.63990272415003,0043-1234,3.202355488525383 +Anne,49,true,162.58644764598222,0043-1234,3.5161591444868794 +Anne,27,false,150.3701008050878,+1 1234,3.9464138286182577 +Dylan,28,false,169.30229830758933,0043-1234,2.3675726838499402 +Charles,27,true,172.7008812661869,0043-1234,2.11768739941611 +Enid,41,true,174.4469279064947,0043-1234,3.2580274779384446 +Francis,43,false,151.28381690862437,+1 1234,1.1314051200623856 +Francis,46,false,160.3626957550438,0043-1234,1.4984921836750873 +Dylan,45,true,168.21770153580078,+1 1234,3.155512041946154 +Francis,40,false,174.60220273433129,0043-1234,0.34609200825472275 diff --git a/testfiles/with-nans.csv b/testfiles/with-nans.csv new file mode 100644 index 0000000..695cf2c --- /dev/null +++ b/testfiles/with-nans.csv @@ -0,0 +1,6 @@ +col1,col2,col3,col4 +hello,1,0.1,true +,2,0.2,false +bye,,0.3,true +test,3,,false +ciao,4,0.4, From e5ab294e3004a1efd97f8dd6bafdeeccd0563ed9 Mon Sep 17 00:00:00 2001 From: "Clelia (Astra) Bertelli" Date: Tue, 7 Apr 2026 23:38:37 +0200 Subject: [PATCH 3/3] chore: drop and fill nans --- .gitignore | 4 +- README.md | 26 ++- __test__/index.spec.ts | 52 ++++- index.d.ts | 2 + src/lib.rs | 97 ++++++++++ testfiles/generated-100.csv | 200 ++++++++++---------- testfiles/{with-nans.csv => with-nulls.csv} | 0 7 files changed, 274 insertions(+), 107 deletions(-) rename testfiles/{with-nans.csv => with-nulls.csv} (100%) diff --git a/.gitignore b/.gitignore index 687533a..bc0c0c7 100644 --- a/.gitignore +++ b/.gitignore @@ -130,7 +130,5 @@ Cargo.lock /npm # test files -testfiles/*.csv +testfiles/generated-*.csv !testfiles/generated-100.csv -!testfiles/err-*-col*.csv -!testfiles/with-nans.csv diff --git a/README.md b/README.md index 649ee13..bef6358 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,16 @@ The `DataFrame` class exposes two methods: - `colDtype`: retrieve the data type of the records contained within a column (integer, float, boolean or string) - `get`: get a column -- `writeCsv`: write the dataframe to CSV (see the next paragraph) +- `writeCsv`: write the dataframe to CSV (see dedicated paragraph) +- `dropNull`/`fillNull`: Drop or fill null values (see dedicated paragraph) +- `dropNan` / `fillNan`: Drop or fill NaN values (see dedicated paragraph) ```typescript const dt = df.colDtype('name') const colData = df.get('name') ``` -Based on the data type of the column, you can use one of the following helper functions to extract the associated array of data (as `string[]`, `boolean[]` or `number[]`): +Based on the data type of the column, you can use one of the following helper functions to extract the associated array of data (as `(string | null)[]`, `(boolean | null)[]` or `(number | null)[]`): ```typescript import { DataType, asBooleanArray, asFloatArray, asIntArray, asStringArray } from '@cle-does-things/sunbears' @@ -104,6 +106,26 @@ hello,1.2,4,true world,2.3,5,false ``` +### Null and NaN dropping and filling + +The DataFrame class supports also methods for filtering out or changing null values in the columns: + +```typescript +const df = readCsv('test.csv') +df.dropNull() // drop null +df.fillNull() // fill null values with the zero value of their type +``` + +> NOTE: `fillNull` works with zero values, which means: `""` for string, `0` for integer and float, `false` for boolean + +You can filter out and change NaN values as well (only applies if there are float-typed columns): + +```typescript +df.dropNan() +df.fillNan() // fill with zero value +df.fillNan(99.3) // fill with a specific value +``` + ## Benchmarking `sunbears` was benchmarked using the `tinybench`-based script you can find [here](./benchmark/bench.ts). The script reports latency statistics related to the `readCsv` and `writeCsv` functions reading/writing increasingly large CSV files (100, 1000, 100.000 and 1.000.000 rows). diff --git a/__test__/index.spec.ts b/__test__/index.spec.ts index 46c992f..b4a4967 100644 --- a/__test__/index.spec.ts +++ b/__test__/index.spec.ts @@ -211,7 +211,7 @@ test('Dropping null values works correctly', (t) => { }) test('Reading null values works correctly', (t) => { - const df = readCsv('testfiles/with-nans.csv') + const df = readCsv('testfiles/with-nulls.csv') t.is(df.len, 5) const col1 = asStringArray(df.get('col1')!)! t.is(col1.filter((x) => x === null).length, 1) @@ -228,7 +228,7 @@ test('Reading null values works correctly', (t) => { }) test('Filling null values works correctly', (t) => { - const df = readCsv('testfiles/with-nans.csv') + const df = readCsv('testfiles/with-nulls.csv') df.fillNull() t.is(df.len, 5) const col1 = asStringArray(df.get('col1')!)! @@ -244,3 +244,51 @@ test('Filling null values works correctly', (t) => { t.is(col4.filter((x) => x === null).length, 0) t.deepEqual(col4, [true, false, true, false, false]) }) + +test('Dropping NaN values works correctly', (t) => { + const col1 = toStringColumn(['hello', 'world', null, 'bye', 'moon', 'hey']) + const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9]) + const col3 = toIntColumn([4, 5, 6, 7, null, 8]) + const col4 = toBoolColumn([true, false, true, false, true, null]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + col3: col3, + col4: col4, + }) + t.is(df.len, 6) + df.dropNan() + t.is(df.len, 5) + t.deepEqual(asStringArray(df.get('col1')!)!, ['hello', 'world', null, 'moon', 'hey']) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 0.5, 8.9]) + t.deepEqual(asIntArray(df.get('col3')!)!, [4, 5, 6, null, 8]) + t.deepEqual(asBooleanArray(df.get('col4')!)!, [true, false, true, true, null]) +}) + +test('Filling NaN values with a specific value works correctly', (t) => { + const col1 = toFloatColumn([1.0, 2.0, NaN, 3.0, 5.0, null]) + const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + }) + t.is(df.len, 6) + df.fillNan(99.3) + t.is(df.len, 6) + t.deepEqual(asFloatArray(df.get('col1')!)!, [1.0, 2.0, 99.3, 3.0, 5.0, null]) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 99.3, 0.5, 8.9]) +}) + +test('Filling NaN values with the zero value works correctly', (t) => { + const col1 = toFloatColumn([1.0, 2.0, NaN, 3.0, 5.0, null]) + const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + }) + t.is(df.len, 6) + df.fillNan() + t.is(df.len, 6) + t.deepEqual(asFloatArray(df.get('col1')!)!, [1.0, 2.0, 0.0, 3.0, 5.0, null]) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 0.0, 0.5, 8.9]) +}) diff --git a/index.d.ts b/index.d.ts index 3d259aa..529dea2 100644 --- a/index.d.ts +++ b/index.d.ts @@ -7,6 +7,8 @@ export declare class DataFrame { colDtype(col: string): DataType | null get(col: string): ColumnData | null get columns(): Record + dropNan(): void + fillNan(fillValue?: number | undefined | null): void dropNull(): void fillNull(): void writeCsv(path: string): void diff --git a/src/lib.rs b/src/lib.rs index 0ed36b0..66e8c8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,6 +168,103 @@ impl DataFrame { self.columns.clone() } + fn get_dtypes(&self) -> Vec { + let mut dtypes = Vec::with_capacity(self.columns.len()); + for k in self.columns().keys() { + let dt = self.col_dtype(k.clone()).unwrap(); + dtypes.push(dt); + } + dtypes + } + + #[napi] + pub fn drop_nan(&mut self) { + let dtypes = self.get_dtypes(); + if dtypes.iter().all(|d| d != &DataType::Float) { + return; + } + let mut idxs = HashSet::new(); + for i in 0..self.len as usize { + let mut has_nan = false; + for col in self.columns.values() { + match col { + ColumnData::Float(f) => { + if f[i].unwrap_or_default().is_nan() { + has_nan = true; + } + } + _ => continue, + } + if has_nan { + break; + } + } + if has_nan { + idxs.insert(i); + } + } + if !idxs.is_empty() { + for col in self.columns.values_mut() { + match col { + ColumnData::Boolean(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Float(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Integer(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::String(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + } + } + self.len -= idxs.len() as u32; + } + } + + #[napi] + pub fn fill_nan(&mut self, fill_value: Option) { + let dtypes = self.get_dtypes(); + if dtypes.iter().all(|d| d != &DataType::Float) { + return; + } + let val = fill_value.unwrap_or_default(); + for col in self.columns.values_mut() { + match col { + ColumnData::Float(f) => { + for item in f.iter_mut().take(self.len as usize) { + if item.unwrap_or_default().is_nan() { + *item = Some(val); + } + } + } + _ => continue, + } + } + } + #[napi] pub fn drop_null(&mut self) { let mut idxs = HashSet::new(); diff --git a/testfiles/generated-100.csv b/testfiles/generated-100.csv index ed14cfb..bdeaab9 100644 --- a/testfiles/generated-100.csv +++ b/testfiles/generated-100.csv @@ -1,101 +1,101 @@ name,age,is_working,height,phone_number,years_of_experience -Dylan,42,false,152.2337643232597,0043-1234,2.710603836108621 -Enid,29,true,169.44529918227406,+1 1234,1.9186595310390577 -Dylan,26,true,155.06276698093373,+1 1234,4.300156885007877 -Anne,21,false,154.98925970803663,0043-1234,1.8933188282167404 -Anne,44,true,180.21502717549743,0043-1234,3.603969658623612 -Bob,40,true,165.40415901447975,+1 1234,2.845444565703957 -Francis,33,false,174.44508280771316,0043-1234,2.9959578228789834 -Dylan,22,false,173.76493458418864,0043-1234,4.829538663354264 -Anne,20,true,178.8647504968728,+1 1234,2.1807362808824395 -Charles,24,false,164.36863959493428,0043-1234,3.5572587149384427 -Enid,30,false,165.6636593782633,+1 1234,3.2930585952088283 -Enid,30,true,171.84380730396535,0043-1234,0.29869230763984844 -Dylan,36,true,155.16495731529344,+1 1234,1.8442913559545064 -Bob,49,false,156.72531783682777,+1 1234,3.9517646609102455 -Anne,28,false,175.16416650994768,+1 1234,0.5932506406862179 -Francis,50,true,167.83948718687446,0043-1234,4.858709505152615 -Bob,32,true,176.05658608461243,0043-1234,3.8678770628403956 -Charles,28,false,157.65302556188283,+1 1234,3.85713775381506 -Francis,31,true,157.05159155275442,0043-1234,3.7941829176287065 -Bob,37,true,157.8242223834105,0043-1234,3.1102010145933776 -Dylan,41,true,152.31187822580569,+1 1234,3.5510443626328847 -Anne,31,false,164.63707496075145,+1 1234,1.8954734566400697 -Charles,49,false,159.6152622470992,+1 1234,1.4710266707931205 -Bob,30,false,173.04776580077328,0043-1234,1.7684680457734039 -Enid,26,true,161.16426180165936,0043-1234,2.5947287988698275 -Bob,33,true,164.584792448029,0043-1234,0.4170630327377409 -Anne,46,true,156.8716989148629,0043-1234,1.9682008452336965 -Charles,28,false,170.92400270139427,+1 1234,3.694392339828717 -Charles,38,false,160.02548944220067,0043-1234,3.7864176354739367 -Enid,38,true,169.11415760888443,+1 1234,2.553769773167301 -Dylan,40,true,156.70256919306888,+1 1234,3.2569541662817443 -Dylan,45,true,178.79902539954585,0043-1234,3.320434569767923 -Dylan,26,true,170.0472625467359,0043-1234,2.3723283322979123 -Francis,49,true,162.2375687558636,+1 1234,4.749812668737068 -Dylan,23,true,156.6866835960943,0043-1234,1.0127174368942098 -Bob,38,true,180.3465341266976,+1 1234,0.35644809979627334 -Enid,36,false,177.68158834314482,+1 1234,1.8603889825422584 -Anne,33,true,180.34367433810976,+1 1234,2.859833382035033 -Charles,49,false,159.06132113458898,+1 1234,2.356082572340613 -Charles,27,false,176.79719018961643,0043-1234,2.361085336622755 -Charles,32,true,165.5038587798262,+1 1234,4.701131180253473 -Bob,50,true,156.2732796282787,+1 1234,2.9871409774632256 -Enid,34,true,168.4792444229309,0043-1234,3.167110474391576 -Bob,41,true,170.65682514988467,+1 1234,3.1842293644017765 -Charles,30,true,180.11440728933985,+1 1234,3.3340682130335937 -Anne,24,true,168.82752873187513,+1 1234,4.074559876799402 -Enid,50,true,173.80312298394057,+1 1234,2.603607644773761 -Dylan,43,true,171.2498164542436,+1 1234,2.995157503942086 -Bob,29,true,176.5332297667723,0043-1234,4.792132532376961 -Enid,24,false,156.3152333573472,+1 1234,2.279840425113064 -Dylan,33,true,164.41672242642258,0043-1234,2.046726377844414 -Charles,30,true,152.98655908018037,+1 1234,3.225492395302485 -Charles,29,true,158.15269781789218,+1 1234,2.232872472543561 -Francis,30,false,170.5090759393567,0043-1234,4.826935025835683 -Francis,34,false,163.93109492473243,0043-1234,1.6240047886100537 -Francis,29,true,172.700773237117,+1 1234,1.3127190206762955 -Dylan,26,true,162.67195903287686,0043-1234,0.33222993830582315 -Charles,47,true,158.14905659054304,+1 1234,1.013882418974281 -Charles,44,true,166.62412538910291,+1 1234,0.045743111899582134 -Anne,28,true,166.95355019282664,+1 1234,0.7982170079180548 -Francis,42,false,156.79758594634148,+1 1234,1.3617907272217888 -Francis,49,true,172.42424362645298,+1 1234,4.342859717299524 -Francis,39,false,161.55673026150154,+1 1234,4.289002272448447 -Dylan,38,true,156.78262933687031,+1 1234,3.2037219868478406 -Charles,31,false,168.94251163149627,0043-1234,2.966612991432437 -Francis,30,false,168.53091362629632,0043-1234,0.27391352355643117 -Dylan,39,true,169.9544991081737,+1 1234,1.0506048994344854 -Francis,29,true,163.69812373179843,0043-1234,4.0952079762730875 -Francis,49,true,163.8229577296399,+1 1234,2.907230653232779 -Enid,38,false,155.63267386685342,0043-1234,2.6336821984845926 -Charles,21,true,171.32125410820913,+1 1234,3.2726028187887195 -Anne,20,false,169.95160147856606,0043-1234,3.636289344725836 -Bob,36,false,171.25000202065186,0043-1234,4.996409278759291 -Bob,20,true,153.68559944681937,0043-1234,4.479337308398978 -Charles,27,false,171.0377797995193,0043-1234,3.3357404123674828 -Enid,36,true,160.66327145682146,0043-1234,2.608196831381723 -Anne,25,false,150.86609499189495,0043-1234,4.523359997234933 -Charles,27,false,161.79640810379877,+1 1234,3.796289654423112 -Francis,46,true,161.16991583857177,0043-1234,2.3019539257627075 -Anne,22,false,177.88531794342396,+1 1234,2.6711824210696182 -Anne,46,true,170.02573691414202,0043-1234,3.3105244025305636 -Charles,30,false,177.56599570674095,0043-1234,0.36264731267778494 -Anne,22,false,174.98008638974463,0043-1234,3.981181990095814 -Francis,38,true,178.3637457193352,+1 1234,3.0846262408736 -Dylan,21,true,159.1454642739654,0043-1234,3.6381043367705663 -Charles,38,false,164.15859475199443,+1 1234,3.0465150075483445 -Anne,21,false,158.56246926321796,+1 1234,2.4885388005852915 -Charles,20,true,151.5147102387871,0043-1234,1.259301200465609 -Enid,47,false,175.338097479699,0043-1234,1.8438560301641096 -Bob,34,false,179.9105013310785,0043-1234,2.0529343125484867 -Francis,21,true,163.63990272415003,0043-1234,3.202355488525383 -Anne,49,true,162.58644764598222,0043-1234,3.5161591444868794 -Anne,27,false,150.3701008050878,+1 1234,3.9464138286182577 -Dylan,28,false,169.30229830758933,0043-1234,2.3675726838499402 -Charles,27,true,172.7008812661869,0043-1234,2.11768739941611 -Enid,41,true,174.4469279064947,0043-1234,3.2580274779384446 -Francis,43,false,151.28381690862437,+1 1234,1.1314051200623856 -Francis,46,false,160.3626957550438,0043-1234,1.4984921836750873 -Dylan,45,true,168.21770153580078,+1 1234,3.155512041946154 -Francis,40,false,174.60220273433129,0043-1234,0.34609200825472275 +Anne,46,true,150.76531309180402,+1 1234,3.954135362601339 +Charles,38,true,175.04914788698258,+1 1234,4.469747450415067 +Charles,25,false,163.67526465841917,0043-1234,3.8173576150877757 +Dylan,43,true,173.38795478611095,+1 1234,0.8386517266869886 +Dylan,37,false,150.39594757953782,+1 1234,3.1870721894508307 +Francis,24,false,158.31265441139837,0043-1234,2.1981097634077855 +Anne,27,false,173.1515407954204,+1 1234,4.136194212596778 +Enid,25,true,156.05074789133388,0043-1234,1.739556269758793 +Anne,39,false,173.52810777068447,+1 1234,1.2489995417268334 +Bob,50,true,153.42886164781913,0043-1234,1.6770132822982462 +Dylan,21,false,179.2522828395681,0043-1234,1.7863819346480414 +Francis,25,true,156.6346853115135,+1 1234,2.042463741009344 +Francis,24,false,158.11615605341967,+1 1234,2.56949178170946 +Dylan,27,true,161.37139485917572,+1 1234,3.9350862773025206 +Francis,31,true,165.57017518941487,0043-1234,3.814675283289935 +Dylan,30,true,162.62835785785694,0043-1234,1.945897387904975 +Dylan,26,false,159.87970172273054,+1 1234,0.06805021607044015 +Anne,45,true,175.94812549101817,+1 1234,1.3893314815728488 +Francis,45,false,175.7362293138813,+1 1234,3.6052931263877808 +Dylan,25,false,151.88137752123708,+1 1234,3.963878639699785 +Bob,40,false,164.54547044859015,0043-1234,0.23957683440593947 +Enid,23,false,174.56554093303052,+1 1234,4.775962996531939 +Charles,45,true,156.40378367695502,0043-1234,1.9013840006588252 +Enid,33,true,170.24919530588542,+1 1234,1.1282698533772666 +Anne,37,false,152.72646295134584,+1 1234,1.2308853135739772 +Enid,44,false,174.4722597943871,+1 1234,3.6275563138173528 +Charles,44,false,159.4958402131124,0043-1234,2.979171609131394 +Charles,21,false,164.73123593787506,0043-1234,3.5656992014674875 +Dylan,33,false,160.7614554541655,+1 1234,3.6679323706625686 +Charles,29,false,163.09159399378922,+1 1234,1.435471780331461 +Bob,35,true,156.06401957178096,0043-1234,2.88383356208846 +Bob,41,true,168.16142115469412,0043-1234,3.468979035606192 +Dylan,29,false,167.09248840740057,0043-1234,3.1696719532545625 +Anne,50,true,178.38402073531012,0043-1234,2.627006237420564 +Dylan,48,true,176.86038496203497,0043-1234,2.7679507347468877 +Charles,23,false,153.18866644041594,0043-1234,1.6844789082793343 +Charles,43,true,151.6886410617161,0043-1234,1.5357351860506172 +Francis,36,true,172.8508080010261,0043-1234,4.418771255213038 +Francis,50,true,167.39583868406044,0043-1234,3.106473063710372 +Francis,42,true,150.41165672386768,+1 1234,0.682512280008466 +Bob,30,true,179.692215206849,+1 1234,1.1121766883161823 +Anne,27,true,172.32634194445856,0043-1234,3.747433966041404 +Bob,44,true,155.4087890966363,0043-1234,1.4052260971732657 +Charles,24,true,167.4607557553078,+1 1234,1.543574982993518 +Dylan,49,false,170.0994907937394,0043-1234,2.7186027840732 +Anne,50,false,178.8254364074925,0043-1234,0.2387747111609675 +Charles,35,true,162.43650998904812,+1 1234,2.8968735593242143 +Enid,30,false,179.9972920567251,+1 1234,0.9246804514269347 +Francis,31,true,155.18363047992236,+1 1234,3.688335106934169 +Anne,25,false,178.5376031748252,+1 1234,3.0880433201079747 +Enid,47,false,159.55714531518137,0043-1234,4.246690984462306 +Francis,47,false,175.1155653303355,+1 1234,0.11798932212723323 +Francis,33,true,179.26573349293375,+1 1234,3.855848137714783 +Charles,32,true,153.98426698533015,+1 1234,4.794466640988527 +Dylan,46,false,167.03836425161518,0043-1234,0.5417002172687309 +Enid,50,true,166.0505680866778,+1 1234,3.0468200378682226 +Anne,27,false,150.2505335972245,0043-1234,2.4575653099174475 +Dylan,25,false,168.45101965152557,0043-1234,4.369140397150075 +Bob,25,false,150.54892843547685,+1 1234,2.0302319350719222 +Bob,32,false,167.59021977381644,+1 1234,1.0875949075469828 +Francis,41,true,152.41336989771224,0043-1234,1.939612300707253 +Bob,41,true,156.6373051059371,+1 1234,1.242528735163106 +Bob,26,true,179.31203897319696,0043-1234,1.204587839355171 +Anne,20,true,160.4432321594598,0043-1234,4.421969739726977 +Dylan,29,true,151.3826655779588,+1 1234,0.6245551975271046 +Dylan,40,false,159.50021636017874,+1 1234,2.025967529155807 +Enid,42,true,156.72444072948502,0043-1234,3.236276945307121 +Anne,47,false,152.83596037349938,0043-1234,1.8199008858359313 +Charles,20,false,168.21226973203548,+1 1234,4.08819523637704 +Francis,42,true,173.82800661014744,+1 1234,3.4152399822214154 +Bob,27,true,164.86422656509552,0043-1234,2.9836280063834666 +Enid,24,false,159.02999953729767,+1 1234,4.455013335430294 +Dylan,34,false,176.16376462869587,0043-1234,4.463097416427798 +Charles,30,true,154.92876753370254,+1 1234,3.140079811827325 +Anne,23,true,180.12588607054136,0043-1234,3.965476039495355 +Charles,27,true,157.4682354799875,0043-1234,2.99883960404666 +Bob,27,false,150.5910029282255,+1 1234,0.358624561860837 +Francis,30,true,168.39443886823884,0043-1234,1.4061994327272518 +Bob,21,false,151.4865633808954,+1 1234,1.9569325065368817 +Bob,34,false,178.79387629130554,+1 1234,2.8145025433680253 +Anne,45,false,179.9005520055787,+1 1234,4.544631930148121 +Dylan,25,false,175.45510848195835,+1 1234,0.3493017501715543 +Charles,35,false,172.08599798708048,+1 1234,1.8001291618592858 +Charles,43,true,158.14664029762915,+1 1234,4.407331458066724 +Charles,38,true,150.04329329341388,0043-1234,0.9104851878769733 +Anne,33,false,176.92056351345093,+1 1234,2.180236882500436 +Charles,41,true,159.5134654013966,0043-1234,2.6839747977657162 +Dylan,28,true,177.84257944745139,+1 1234,2.5258626507548168 +Enid,25,true,180.95968401074447,+1 1234,4.513885416532654 +Anne,38,true,175.81483444931558,+1 1234,3.5682470398184436 +Anne,27,true,169.40340812938422,+1 1234,1.1624130820711818 +Dylan,27,false,173.4171445681396,0043-1234,2.0519091319479728 +Francis,28,false,160.84177820755508,+1 1234,1.8893825837047076 +Bob,26,false,161.35782863798133,0043-1234,4.152298079763579 +Bob,34,false,161.29137062655272,0043-1234,3.83678871436004 +Enid,32,true,179.8387575766374,0043-1234,1.0064266063376694 +Dylan,26,true,154.3169568388267,0043-1234,2.11365904576754 +Francis,40,true,173.9416162246519,0043-1234,4.62788887546622 +Anne,25,true,172.42133367379003,+1 1234,2.8577621288636665 +Anne,38,false,156.3816430044022,+1 1234,2.936138661291708 diff --git a/testfiles/with-nans.csv b/testfiles/with-nulls.csv similarity index 100% rename from testfiles/with-nans.csv rename to testfiles/with-nulls.csv