wip: add support for nan

.gitignore

@@ -130,6 +130,5 @@ Cargo.lock
 /npm
 
 # test files
-testfiles/*.csv
+testfiles/generated-*.csv
 !testfiles/generated-100.csv
-!testfiles/err-*-col*.csv

README.md

@@ -26,14 +26,16 @@ The `DataFrame` class exposes two methods:
 
 - `colDtype`: retrieve the data type of the records contained within a column (integer, float, boolean or string)
 - `get`: get a column
-- `writeCsv`: write the dataframe to CSV (see the next paragraph)
+- `writeCsv`: write the dataframe to CSV (see dedicated paragraph)
+- `dropNull`/`fillNull`: Drop or fill null values (see dedicated paragraph)
+- `dropNan` / `fillNan`: Drop or fill NaN values (see dedicated paragraph)
 
 ```typescript
 const dt = df.colDtype('name')
 const colData = df.get('name')
 ```
 
-Based on the data type of the column, you can use one of the following helper functions to extract the associated array of data (as `string[]`, `boolean[]` or `number[]`):
+Based on the data type of the column, you can use one of the following helper functions to extract the associated array of data (as `(string | null)[]`, `(boolean | null)[]` or `(number | null)[]`):
 
 ```typescript
 import { DataType, asBooleanArray, asFloatArray, asIntArray, asStringArray } from '@cle-does-things/sunbears'
@@ -104,6 +106,26 @@ hello,1.2,4,true
 world,2.3,5,false
 ```
 
+### Null and NaN dropping and filling
+
+The DataFrame class supports also methods for filtering out or changing null values in the columns:
+
+```typescript
+const df = readCsv('test.csv')
+df.dropNull() // drop null
+df.fillNull() // fill null values with the zero value of their type
+```
+
+> NOTE: `fillNull` works with zero values, which means: `""` for string, `0` for integer and float, `false` for boolean
+
+You can filter out and change NaN values as well (only applies if there are float-typed columns):
+
+```typescript
+df.dropNan()
+df.fillNan() // fill with zero value
+df.fillNan(99.3) // fill with a specific value
+```
+
 ## Benchmarking
 
 `sunbears` was benchmarked using the `tinybench`-based script you can find [here](./benchmark/bench.ts). The script reports latency statistics related to the `readCsv` and `writeCsv` functions reading/writing increasingly large CSV files (100, 1000, 100.000 and 1.000.000 rows).

__test__/index.spec.ts

@@ -189,3 +189,106 @@ test('Write to CSV works correctly', (t) => {
   }
   unlinkSync('testfiles/write-test.csv')
 })
+
+test('Dropping null values works correctly', (t) => {
+  const col1 = toStringColumn(['hello', 'world', null, 'bye', 'moon', 'hey'])
+  const col2 = toFloatColumn([1.2, 2.3, 0.3, null, 0.5, 8.9])
+  const col3 = toIntColumn([4, 5, 6, 7, null, 8])
+  const col4 = toBoolColumn([true, false, true, false, true, null])
+  const df = DataFrame.fromColumns({
+    col1: col1,
+    col2: col2,
+    col3: col3,
+    col4: col4,
+  })
+  t.is(df.len, 6)
+  df.dropNull()
+  t.is(df.len, 2)
+  t.deepEqual(asStringArray(df.get('col1')!)!, ['hello', 'world'])
+  t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3])
+  t.deepEqual(asIntArray(df.get('col3')!)!, [4, 5])
+  t.deepEqual(asBooleanArray(df.get('col4')!)!, [true, false])
+})
+
+test('Reading null values works correctly', (t) => {
+  const df = readCsv('testfiles/with-nulls.csv')
+  t.is(df.len, 5)
+  const col1 = asStringArray(df.get('col1')!)!
+  t.is(col1.filter((x) => x === null).length, 1)
+  t.deepEqual(col1, ['hello', null, 'bye', 'test', 'ciao'])
+  const col2 = asIntArray(df.get('col2')!)!
+  t.is(col2.filter((x) => x === null).length, 1)
+  t.deepEqual(col2, [1, 2, null, 3, 4])
+  const col3 = asFloatArray(df.get('col3')!)!
+  t.is(col3.filter((x) => x === null).length, 1)
+  t.deepEqual(col3, [0.1, 0.2, 0.3, null, 0.4])
+  const col4 = asBooleanArray(df.get('col4')!)!
+  t.is(col4.filter((x) => x === null).length, 1)
+  t.deepEqual(col4, [true, false, true, false, null])
+})
+
+test('Filling null values works correctly', (t) => {
+  const df = readCsv('testfiles/with-nulls.csv')
+  df.fillNull()
+  t.is(df.len, 5)
+  const col1 = asStringArray(df.get('col1')!)!
+  t.is(col1.filter((x) => x === null).length, 0)
+  t.deepEqual(col1, ['hello', '', 'bye', 'test', 'ciao'])
+  const col2 = asIntArray(df.get('col2')!)!
+  t.is(col2.filter((x) => x === null).length, 0)
+  t.deepEqual(col2, [1, 2, 0, 3, 4])
+  const col3 = asFloatArray(df.get('col3')!)!
+  t.is(col3.filter((x) => x === null).length, 0)
+  t.deepEqual(col3, [0.1, 0.2, 0.3, 0.0, 0.4])
+  const col4 = asBooleanArray(df.get('col4')!)!
+  t.is(col4.filter((x) => x === null).length, 0)
+  t.deepEqual(col4, [true, false, true, false, false])
+})
+
+test('Dropping NaN values works correctly', (t) => {
+  const col1 = toStringColumn(['hello', 'world', null, 'bye', 'moon', 'hey'])
+  const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9])
+  const col3 = toIntColumn([4, 5, 6, 7, null, 8])
+  const col4 = toBoolColumn([true, false, true, false, true, null])
+  const df = DataFrame.fromColumns({
+    col1: col1,
+    col2: col2,
+    col3: col3,
+    col4: col4,
+  })
+  t.is(df.len, 6)
+  df.dropNan()
+  t.is(df.len, 5)
+  t.deepEqual(asStringArray(df.get('col1')!)!, ['hello', 'world', null, 'moon', 'hey'])
+  t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 0.5, 8.9])
+  t.deepEqual(asIntArray(df.get('col3')!)!, [4, 5, 6, null, 8])
+  t.deepEqual(asBooleanArray(df.get('col4')!)!, [true, false, true, true, null])
+})
+
+test('Filling NaN values with a specific value works correctly', (t) => {
+  const col1 = toFloatColumn([1.0, 2.0, NaN, 3.0, 5.0, null])
+  const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9])
+  const df = DataFrame.fromColumns({
+    col1: col1,
+    col2: col2,
+  })
+  t.is(df.len, 6)
+  df.fillNan(99.3)
+  t.is(df.len, 6)
+  t.deepEqual(asFloatArray(df.get('col1')!)!, [1.0, 2.0, 99.3, 3.0, 5.0, null])
+  t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 99.3, 0.5, 8.9])
+})
+
+test('Filling NaN values with the zero value works correctly', (t) => {
+  const col1 = toFloatColumn([1.0, 2.0, NaN, 3.0, 5.0, null])
+  const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9])
+  const df = DataFrame.fromColumns({
+    col1: col1,
+    col2: col2,
+  })
+  t.is(df.len, 6)
+  df.fillNan()
+  t.is(df.len, 6)
+  t.deepEqual(asFloatArray(df.get('col1')!)!, [1.0, 2.0, 0.0, 3.0, 5.0, null])
+  t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 0.0, 0.5, 8.9])
+})

index.d.ts

@@ -7,22 +7,26 @@ export declare class DataFrame {
   colDtype(col: string): DataType | null
   get(col: string): ColumnData | null
   get columns(): Record<string, ColumnData>
+  dropNan(): void
+  fillNan(fillValue?: number | undefined | null): void
+  dropNull(): void
+  fillNull(): void
   writeCsv(path: string): void
 }
 
-export declare function asBooleanArray(column: ColumnData): Array<boolean> | null
+export declare function asBooleanArray(column: ColumnData): Array<boolean | undefined | null> | null
 
-export declare function asFloatArray(column: ColumnData): Array<number> | null
+export declare function asFloatArray(column: ColumnData): Array<number | undefined | null> | null
 
-export declare function asIntArray(column: ColumnData): Array<number> | null
+export declare function asIntArray(column: ColumnData): Array<number | undefined | null> | null
 
-export declare function asStringArray(column: ColumnData): Array<string> | null
+export declare function asStringArray(column: ColumnData): Array<string | undefined | null> | null
 
 export type ColumnData =
-  | { type: 'String'; field0: Array<string> }
-  | { type: 'Integer'; field0: Array<number> }
-  | { type: 'Float'; field0: Array<number> }
-  | { type: 'Boolean'; field0: Array<boolean> }
+  | { type: 'String'; field0: Array<string | undefined | null> }
+  | { type: 'Integer'; field0: Array<number | undefined | null> }
+  | { type: 'Float'; field0: Array<number | undefined | null> }
+  | { type: 'Boolean'; field0: Array<boolean | undefined | null> }
 
 export declare const enum DataType {
   String = 0,
@@ -33,10 +37,10 @@ export declare const enum DataType {
 
 export declare function readCsv(path: string): DataFrame
 
-export declare function toBoolColumn(data: Array<boolean>): ColumnData
+export declare function toBoolColumn(data: Array<boolean | undefined | null>): ColumnData
 
-export declare function toFloatColumn(data: Array<number>): ColumnData
+export declare function toFloatColumn(data: Array<number | undefined | null>): ColumnData
 
-export declare function toIntColumn(data: Array<number>): ColumnData
+export declare function toIntColumn(data: Array<number | undefined | null>): ColumnData
 
-export declare function toStringColumn(data: Array<string>): ColumnData
+export declare function toStringColumn(data: Array<string | undefined | null>): ColumnData