diff --git a/.gitignore b/.gitignore index 0ec9a24..bc0c0c7 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,5 @@ Cargo.lock /npm # test files -testfiles/*.csv +testfiles/generated-*.csv !testfiles/generated-100.csv -!testfiles/err-*-col*.csv diff --git a/README.md b/README.md index 649ee13..bef6358 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,16 @@ The `DataFrame` class exposes two methods: - `colDtype`: retrieve the data type of the records contained within a column (integer, float, boolean or string) - `get`: get a column -- `writeCsv`: write the dataframe to CSV (see the next paragraph) +- `writeCsv`: write the dataframe to CSV (see dedicated paragraph) +- `dropNull`/`fillNull`: Drop or fill null values (see dedicated paragraph) +- `dropNan` / `fillNan`: Drop or fill NaN values (see dedicated paragraph) ```typescript const dt = df.colDtype('name') const colData = df.get('name') ``` -Based on the data type of the column, you can use one of the following helper functions to extract the associated array of data (as `string[]`, `boolean[]` or `number[]`): +Based on the data type of the column, you can use one of the following helper functions to extract the associated array of data (as `(string | null)[]`, `(boolean | null)[]` or `(number | null)[]`): ```typescript import { DataType, asBooleanArray, asFloatArray, asIntArray, asStringArray } from '@cle-does-things/sunbears' @@ -104,6 +106,26 @@ hello,1.2,4,true world,2.3,5,false ``` +### Null and NaN dropping and filling + +The DataFrame class supports also methods for filtering out or changing null values in the columns: + +```typescript +const df = readCsv('test.csv') +df.dropNull() // drop null +df.fillNull() // fill null values with the zero value of their type +``` + +> NOTE: `fillNull` works with zero values, which means: `""` for string, `0` for integer and float, `false` for boolean + +You can filter out and change NaN values as well (only applies if there are float-typed columns): + +```typescript +df.dropNan() +df.fillNan() // fill with zero value +df.fillNan(99.3) // fill with a specific value +``` + ## Benchmarking `sunbears` was benchmarked using the `tinybench`-based script you can find [here](./benchmark/bench.ts). The script reports latency statistics related to the `readCsv` and `writeCsv` functions reading/writing increasingly large CSV files (100, 1000, 100.000 and 1.000.000 rows). diff --git a/__test__/index.spec.ts b/__test__/index.spec.ts index 3f62df5..b4a4967 100644 --- a/__test__/index.spec.ts +++ b/__test__/index.spec.ts @@ -189,3 +189,106 @@ test('Write to CSV works correctly', (t) => { } unlinkSync('testfiles/write-test.csv') }) + +test('Dropping null values works correctly', (t) => { + const col1 = toStringColumn(['hello', 'world', null, 'bye', 'moon', 'hey']) + const col2 = toFloatColumn([1.2, 2.3, 0.3, null, 0.5, 8.9]) + const col3 = toIntColumn([4, 5, 6, 7, null, 8]) + const col4 = toBoolColumn([true, false, true, false, true, null]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + col3: col3, + col4: col4, + }) + t.is(df.len, 6) + df.dropNull() + t.is(df.len, 2) + t.deepEqual(asStringArray(df.get('col1')!)!, ['hello', 'world']) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3]) + t.deepEqual(asIntArray(df.get('col3')!)!, [4, 5]) + t.deepEqual(asBooleanArray(df.get('col4')!)!, [true, false]) +}) + +test('Reading null values works correctly', (t) => { + const df = readCsv('testfiles/with-nulls.csv') + t.is(df.len, 5) + const col1 = asStringArray(df.get('col1')!)! + t.is(col1.filter((x) => x === null).length, 1) + t.deepEqual(col1, ['hello', null, 'bye', 'test', 'ciao']) + const col2 = asIntArray(df.get('col2')!)! + t.is(col2.filter((x) => x === null).length, 1) + t.deepEqual(col2, [1, 2, null, 3, 4]) + const col3 = asFloatArray(df.get('col3')!)! + t.is(col3.filter((x) => x === null).length, 1) + t.deepEqual(col3, [0.1, 0.2, 0.3, null, 0.4]) + const col4 = asBooleanArray(df.get('col4')!)! + t.is(col4.filter((x) => x === null).length, 1) + t.deepEqual(col4, [true, false, true, false, null]) +}) + +test('Filling null values works correctly', (t) => { + const df = readCsv('testfiles/with-nulls.csv') + df.fillNull() + t.is(df.len, 5) + const col1 = asStringArray(df.get('col1')!)! + t.is(col1.filter((x) => x === null).length, 0) + t.deepEqual(col1, ['hello', '', 'bye', 'test', 'ciao']) + const col2 = asIntArray(df.get('col2')!)! + t.is(col2.filter((x) => x === null).length, 0) + t.deepEqual(col2, [1, 2, 0, 3, 4]) + const col3 = asFloatArray(df.get('col3')!)! + t.is(col3.filter((x) => x === null).length, 0) + t.deepEqual(col3, [0.1, 0.2, 0.3, 0.0, 0.4]) + const col4 = asBooleanArray(df.get('col4')!)! + t.is(col4.filter((x) => x === null).length, 0) + t.deepEqual(col4, [true, false, true, false, false]) +}) + +test('Dropping NaN values works correctly', (t) => { + const col1 = toStringColumn(['hello', 'world', null, 'bye', 'moon', 'hey']) + const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9]) + const col3 = toIntColumn([4, 5, 6, 7, null, 8]) + const col4 = toBoolColumn([true, false, true, false, true, null]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + col3: col3, + col4: col4, + }) + t.is(df.len, 6) + df.dropNan() + t.is(df.len, 5) + t.deepEqual(asStringArray(df.get('col1')!)!, ['hello', 'world', null, 'moon', 'hey']) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 0.5, 8.9]) + t.deepEqual(asIntArray(df.get('col3')!)!, [4, 5, 6, null, 8]) + t.deepEqual(asBooleanArray(df.get('col4')!)!, [true, false, true, true, null]) +}) + +test('Filling NaN values with a specific value works correctly', (t) => { + const col1 = toFloatColumn([1.0, 2.0, NaN, 3.0, 5.0, null]) + const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + }) + t.is(df.len, 6) + df.fillNan(99.3) + t.is(df.len, 6) + t.deepEqual(asFloatArray(df.get('col1')!)!, [1.0, 2.0, 99.3, 3.0, 5.0, null]) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 99.3, 0.5, 8.9]) +}) + +test('Filling NaN values with the zero value works correctly', (t) => { + const col1 = toFloatColumn([1.0, 2.0, NaN, 3.0, 5.0, null]) + const col2 = toFloatColumn([1.2, 2.3, 0.3, NaN, 0.5, 8.9]) + const df = DataFrame.fromColumns({ + col1: col1, + col2: col2, + }) + t.is(df.len, 6) + df.fillNan() + t.is(df.len, 6) + t.deepEqual(asFloatArray(df.get('col1')!)!, [1.0, 2.0, 0.0, 3.0, 5.0, null]) + t.deepEqual(asFloatArray(df.get('col2')!)!, [1.2, 2.3, 0.3, 0.0, 0.5, 8.9]) +}) diff --git a/index.d.ts b/index.d.ts index 25b9470..529dea2 100644 --- a/index.d.ts +++ b/index.d.ts @@ -7,22 +7,26 @@ export declare class DataFrame { colDtype(col: string): DataType | null get(col: string): ColumnData | null get columns(): Record + dropNan(): void + fillNan(fillValue?: number | undefined | null): void + dropNull(): void + fillNull(): void writeCsv(path: string): void } -export declare function asBooleanArray(column: ColumnData): Array | null +export declare function asBooleanArray(column: ColumnData): Array | null -export declare function asFloatArray(column: ColumnData): Array | null +export declare function asFloatArray(column: ColumnData): Array | null -export declare function asIntArray(column: ColumnData): Array | null +export declare function asIntArray(column: ColumnData): Array | null -export declare function asStringArray(column: ColumnData): Array | null +export declare function asStringArray(column: ColumnData): Array | null export type ColumnData = - | { type: 'String'; field0: Array } - | { type: 'Integer'; field0: Array } - | { type: 'Float'; field0: Array } - | { type: 'Boolean'; field0: Array } + | { type: 'String'; field0: Array } + | { type: 'Integer'; field0: Array } + | { type: 'Float'; field0: Array } + | { type: 'Boolean'; field0: Array } export declare const enum DataType { String = 0, @@ -33,10 +37,10 @@ export declare const enum DataType { export declare function readCsv(path: string): DataFrame -export declare function toBoolColumn(data: Array): ColumnData +export declare function toBoolColumn(data: Array): ColumnData -export declare function toFloatColumn(data: Array): ColumnData +export declare function toFloatColumn(data: Array): ColumnData -export declare function toIntColumn(data: Array): ColumnData +export declare function toIntColumn(data: Array): ColumnData -export declare function toStringColumn(data: Array): ColumnData +export declare function toStringColumn(data: Array): ColumnData diff --git a/index.js b/index.js index c53cb9c..42b8acf 100644 --- a/index.js +++ b/index.js @@ -77,8 +77,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-android-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-android-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -93,8 +93,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-android-arm-eabi') const bindingPackageVersion = require('@cle-does-things/sunbears-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -114,8 +114,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-x64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-x64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -130,8 +130,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-x64-msvc') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -147,8 +147,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-ia32-msvc') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -163,8 +163,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-win32-arm64-msvc') const bindingPackageVersion = require('@cle-does-things/sunbears-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -182,8 +182,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-darwin-universal') const bindingPackageVersion = require('@cle-does-things/sunbears-darwin-universal/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -198,8 +198,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-darwin-x64') const bindingPackageVersion = require('@cle-does-things/sunbears-darwin-x64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -214,8 +214,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-darwin-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-darwin-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -234,8 +234,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-freebsd-x64') const bindingPackageVersion = require('@cle-does-things/sunbears-freebsd-x64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -250,8 +250,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-freebsd-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -271,8 +271,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-x64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -287,8 +287,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-x64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -305,8 +305,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -321,8 +321,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -339,8 +339,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm-musleabihf') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -355,8 +355,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-arm-gnueabihf') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -373,8 +373,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-loong64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-loong64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -389,8 +389,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-loong64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-loong64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -407,8 +407,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-riscv64-musl') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -423,8 +423,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-riscv64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -440,8 +440,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-ppc64-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -456,8 +456,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-linux-s390x-gnu') const bindingPackageVersion = require('@cle-does-things/sunbears-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -476,8 +476,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-openharmony-arm64') const bindingPackageVersion = require('@cle-does-things/sunbears-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -492,8 +492,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-openharmony-x64') const bindingPackageVersion = require('@cle-does-things/sunbears-openharmony-x64/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -508,8 +508,8 @@ function requireNative() { try { const binding = require('@cle-does-things/sunbears-openharmony-arm') const bindingPackageVersion = require('@cle-does-things/sunbears-openharmony-arm/package.json').version - if (bindingPackageVersion !== '1.0.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 1.0.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '1.1.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 1.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { diff --git a/src/lib.rs b/src/lib.rs index 3e7c43c..66e8c8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,11 @@ #![deny(clippy::all)] -use std::{collections::HashMap, fmt, fs::File, io::BufWriter}; +use std::{ + collections::{HashMap, HashSet}, + fmt, + fs::File, + io::BufWriter, +}; use anyhow::{anyhow, Result}; @@ -12,10 +17,10 @@ use napi_derive::napi; #[napi] #[derive(Clone)] pub enum ColumnData { - String(Vec), - Integer(Vec), - Float(Vec), - Boolean(Vec), + String(Vec>), + Integer(Vec>), + Float(Vec>), + Boolean(Vec>), } impl ColumnData { @@ -53,7 +58,7 @@ impl fmt::Display for DataType { // ---------------------------------------- HELPER FUNCTIONS FOR DATA MODELS ---------------------------------------- #[napi] -pub fn as_float_array(column: ColumnData) -> Option> { +pub fn as_float_array(column: ColumnData) -> Option>> { match column { ColumnData::Float(f) => Some(f), _ => None, @@ -61,7 +66,7 @@ pub fn as_float_array(column: ColumnData) -> Option> { } #[napi] -pub fn as_int_array(column: ColumnData) -> Option> { +pub fn as_int_array(column: ColumnData) -> Option>> { match column { ColumnData::Integer(f) => Some(f), _ => None, @@ -69,7 +74,7 @@ pub fn as_int_array(column: ColumnData) -> Option> { } #[napi] -pub fn as_boolean_array(column: ColumnData) -> Option> { +pub fn as_boolean_array(column: ColumnData) -> Option>> { match column { ColumnData::Boolean(f) => Some(f), _ => None, @@ -77,7 +82,7 @@ pub fn as_boolean_array(column: ColumnData) -> Option> { } #[napi] -pub fn as_string_array(column: ColumnData) -> Option> { +pub fn as_string_array(column: ColumnData) -> Option>> { match column { ColumnData::String(f) => Some(f), _ => None, @@ -85,22 +90,22 @@ pub fn as_string_array(column: ColumnData) -> Option> { } #[napi] -pub fn to_string_column(data: Vec) -> ColumnData { +pub fn to_string_column(data: Vec>) -> ColumnData { ColumnData::String(data) } #[napi] -pub fn to_float_column(data: Vec) -> ColumnData { +pub fn to_float_column(data: Vec>) -> ColumnData { ColumnData::Float(data) } #[napi] -pub fn to_int_column(data: Vec) -> ColumnData { +pub fn to_int_column(data: Vec>) -> ColumnData { ColumnData::Integer(data) } #[napi] -pub fn to_bool_column(data: Vec) -> ColumnData { +pub fn to_bool_column(data: Vec>) -> ColumnData { ColumnData::Boolean(data) } @@ -163,6 +168,216 @@ impl DataFrame { self.columns.clone() } + fn get_dtypes(&self) -> Vec { + let mut dtypes = Vec::with_capacity(self.columns.len()); + for k in self.columns().keys() { + let dt = self.col_dtype(k.clone()).unwrap(); + dtypes.push(dt); + } + dtypes + } + + #[napi] + pub fn drop_nan(&mut self) { + let dtypes = self.get_dtypes(); + if dtypes.iter().all(|d| d != &DataType::Float) { + return; + } + let mut idxs = HashSet::new(); + for i in 0..self.len as usize { + let mut has_nan = false; + for col in self.columns.values() { + match col { + ColumnData::Float(f) => { + if f[i].unwrap_or_default().is_nan() { + has_nan = true; + } + } + _ => continue, + } + if has_nan { + break; + } + } + if has_nan { + idxs.insert(i); + } + } + if !idxs.is_empty() { + for col in self.columns.values_mut() { + match col { + ColumnData::Boolean(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Float(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Integer(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::String(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + } + } + self.len -= idxs.len() as u32; + } + } + + #[napi] + pub fn fill_nan(&mut self, fill_value: Option) { + let dtypes = self.get_dtypes(); + if dtypes.iter().all(|d| d != &DataType::Float) { + return; + } + let val = fill_value.unwrap_or_default(); + for col in self.columns.values_mut() { + match col { + ColumnData::Float(f) => { + for item in f.iter_mut().take(self.len as usize) { + if item.unwrap_or_default().is_nan() { + *item = Some(val); + } + } + } + _ => continue, + } + } + } + + #[napi] + pub fn drop_null(&mut self) { + let mut idxs = HashSet::new(); + for i in 0..self.len as usize { + let mut has_none = false; + for col in self.columns.values() { + match col { + ColumnData::Boolean(b) => { + if b[i].is_none() { + has_none = true; + } + } + ColumnData::Float(b) => { + if b[i].is_none() { + has_none = true; + } + } + ColumnData::String(b) => { + if b[i].is_none() { + has_none = true; + } + } + ColumnData::Integer(b) => { + if b[i].is_none() { + has_none = true; + } + } + } + if has_none { + break; + } + } + if has_none { + idxs.insert(i); + } + } + if !idxs.is_empty() { + for col in self.columns.values_mut() { + match col { + ColumnData::Boolean(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Float(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::Integer(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + ColumnData::String(b) => { + let mut i = 0; + b.retain(|_| { + let keep = !idxs.contains(&i); + i += 1; + keep + }); + } + } + } + self.len -= idxs.len() as u32; + } + } + + #[napi] + pub fn fill_null(&mut self) { + for col in self.columns.values_mut() { + match col { + ColumnData::Boolean(b) => { + for item in b.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(false); + } + } + } + ColumnData::Float(f) => { + for item in f.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(0_f64); + } + } + } + ColumnData::String(s) => { + for item in s.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(String::new()); + } + } + } + ColumnData::Integer(j) => { + for item in j.iter_mut().take(self.len as usize) { + if item.is_none() { + *item = Some(0_i64); + } + } + } + } + } + } + #[napi] pub fn write_csv(&self, path: String) -> Result<()> { let file = File::create(&path)?; @@ -178,7 +393,7 @@ impl DataFrame { let col = &self.columns[*key]; match col { ColumnData::Boolean(b) => { - let v = if b[i] { + let v = if b[i].unwrap_or_default() { "true".to_string() } else { "false".to_string() @@ -186,15 +401,15 @@ impl DataFrame { row.push(v); } ColumnData::Float(f) => { - let v = float_buf.format(f[i]).to_string(); + let v = float_buf.format(f[i].unwrap_or_default()).to_string(); row.push(v); } ColumnData::Integer(j) => { - let v = int_buf.format(j[i]).to_string(); + let v = int_buf.format(j[i].unwrap_or_default()).to_string(); row.push(v); } ColumnData::String(s) => { - row.push(s[i].clone()); + row.push(s[i].clone().unwrap_or_default()); } } } @@ -265,10 +480,14 @@ pub fn read_csv(path: String) -> Result { let vc = &col_to_vec[*c]; let data: ColumnData = match d { DataType::Boolean => { - let mut v: Vec = vec![]; + let mut v: Vec> = Vec::with_capacity(vc.len()); for el in vc { + if el.is_empty() { + v.push(None); + continue; + } match str_to_bool(el) { - Some(val) => v.push(val), + Some(val) => v.push(Some(val)), None => { return Err(anyhow!( "Expecting bool at line {:?} for col {}", @@ -281,10 +500,14 @@ pub fn read_csv(path: String) -> Result { ColumnData::Boolean(v) } DataType::Float => { - let mut v: Vec = vec![]; + let mut v: Vec> = Vec::with_capacity(vc.len()); for s in vc { + if s.is_empty() { + v.push(None); + continue; + } match s.parse::() { - Ok(val) => v.push(val), + Ok(val) => v.push(Some(val)), Err(_) => { return Err(anyhow!( "Expecting float at line {:?} for col {}", @@ -298,14 +521,25 @@ pub fn read_csv(path: String) -> Result { ColumnData::Float(v) } DataType::String => { - let v = vc.iter().map(|s| s.to_string()).collect(); + let mut v: Vec> = Vec::with_capacity(vc.len()); + for s in vc { + if s.is_empty() { + v.push(None); + continue; + } + v.push(Some(s.to_string())); + } ColumnData::String(v) } DataType::Integer => { - let mut v: Vec = vec![]; + let mut v: Vec> = Vec::with_capacity(vc.len()); for s in vc { + if s.is_empty() { + v.push(None); + continue; + } match s.parse::() { - Ok(val) => v.push(val), + Ok(val) => v.push(Some(val)), Err(_) => { return Err(anyhow!( "Expecting integer at line {:?} for col {}", diff --git a/testfiles/generated-100.csv b/testfiles/generated-100.csv index ea61a40..bdeaab9 100644 --- a/testfiles/generated-100.csv +++ b/testfiles/generated-100.csv @@ -1,101 +1,101 @@ name,age,is_working,height,phone_number,years_of_experience -Anne,24,false,172.06195385839587,+1 1234,3.767044082274923 -Enid,36,true,164.17546867223956,+1 1234,3.9987730641281765 -Francis,45,true,167.66954681406762,0043-1234,0.4383413027655949 -Enid,34,true,161.30458950930952,+1 1234,1.0983771829465512 -Charles,25,false,151.67913999785412,0043-1234,2.434259647739991 -Dylan,31,true,161.16597974594205,0043-1234,3.276539501458718 -Charles,43,false,153.78110220498175,+1 1234,4.134995277849777 -Charles,25,true,166.40188604691568,0043-1234,3.4263596384326083 -Bob,23,false,158.38593329361092,0043-1234,1.0320864141187804 -Bob,22,false,174.30421224335603,0043-1234,0.30139282444920124 -Charles,25,false,167.36986172098074,+1 1234,3.6070863321158333 -Bob,47,true,155.77584934965734,+1 1234,0.3358753495974762 -Anne,47,true,152.07245609048525,0043-1234,3.978476753629136 -Anne,35,false,163.8284506948372,0043-1234,4.7375086847218375 -Anne,49,true,180.1237482982292,+1 1234,1.4734837130167757 -Francis,31,false,171.39476207206496,+1 1234,2.057482207163191 -Charles,20,false,150.85891722659494,+1 1234,1.8076413190188547 -Charles,26,true,157.73556835468887,0043-1234,3.2390834201030048 -Bob,28,false,180.22280208552263,0043-1234,0.11382435925677648 -Francis,50,true,162.39526970619806,+1 1234,0.8249470790353828 -Charles,39,true,154.01212102360162,+1 1234,1.6846034197031907 -Francis,27,true,173.5486689216849,+1 1234,3.9985312398191337 -Francis,23,false,177.5849194905182,0043-1234,2.6625711310679345 -Francis,20,true,176.23107677217902,+1 1234,2.36102935780334 -Francis,34,true,180.7011235918709,0043-1234,1.482859764730454 -Anne,28,true,172.2611271479767,+1 1234,0.8026755462524306 -Francis,35,true,163.02339898361976,+1 1234,0.13368333645175268 -Dylan,21,false,164.56067064798725,+1 1234,1.1321786041675934 -Anne,41,true,179.442405598062,0043-1234,2.4910230451061053 -Dylan,40,true,169.0704375228679,+1 1234,3.767070661054649 -Bob,45,false,158.52554354403676,0043-1234,3.389188917679733 -Anne,49,true,178.3749017440682,+1 1234,1.2717571652842483 -Francis,42,false,179.89635183986815,+1 1234,0.7228721084532994 -Francis,50,false,162.16345519279804,0043-1234,2.1467121727357386 -Anne,25,false,151.85714506267925,0043-1234,2.0106703515331454 -Enid,43,true,154.44252150760803,0043-1234,0.10012231117163695 -Francis,33,false,169.75224887442954,0043-1234,2.1155764577654237 -Dylan,21,true,163.42614142253774,+1 1234,4.1636259380519896 -Charles,39,false,177.19248309034617,0043-1234,0.7043319312506218 -Dylan,27,false,163.78268872264073,0043-1234,0.7933675094289101 -Enid,22,true,151.69475029878475,0043-1234,3.427832466709869 -Anne,24,true,167.0624790612009,0043-1234,3.208757157964802 -Enid,22,true,161.48780558514278,+1 1234,1.8090922295976357 -Francis,49,true,161.18349998788094,+1 1234,1.0264327081269466 -Enid,30,true,155.8820687964176,0043-1234,1.0565103129209166 -Anne,50,false,163.58681127771385,0043-1234,2.834026981707148 -Enid,25,true,165.3165808966044,0043-1234,2.907435266125858 -Dylan,33,true,178.4105287916339,0043-1234,3.1364447532695845 -Francis,24,false,153.3751836951103,0043-1234,3.573562444922654 -Francis,33,true,175.3551671500021,0043-1234,0.8777011127810928 -Bob,28,false,167.79756345857308,0043-1234,2.5037076919560795 -Dylan,31,true,151.22287551814932,+1 1234,3.909823087240767 -Anne,24,true,159.32157999459298,0043-1234,3.304525677644712 -Dylan,45,false,179.506794367215,0043-1234,1.94228733904784 -Enid,47,false,153.09902751239227,0043-1234,1.2257610300558768 -Enid,24,false,155.81295425867603,0043-1234,2.5198447985382337 -Dylan,35,true,177.8206091922323,0043-1234,3.11915384637018 -Charles,36,false,169.68986298923735,+1 1234,2.040447345083472 -Bob,26,true,175.03218264042204,+1 1234,0.969361969266481 -Bob,33,false,172.40702314693513,0043-1234,3.0529996726430175 -Dylan,21,true,180.24598488538015,+1 1234,2.4544469347421005 -Charles,47,false,152.5271909335372,+1 1234,4.862982108147397 -Bob,31,true,174.82341433179184,+1 1234,4.772525888375782 -Dylan,47,false,159.68297409447837,0043-1234,2.910096160363199 -Dylan,26,false,163.5897866306247,0043-1234,4.5364911850626335 -Dylan,23,true,150.732788122647,0043-1234,4.315366231031446 -Enid,23,false,171.53990655114706,+1 1234,0.5693450705816476 -Enid,48,false,163.53419005719897,0043-1234,1.2921866236585322 -Francis,47,false,151.76618196211368,+1 1234,2.3720314158733604 -Charles,27,true,159.52354929775623,+1 1234,4.867960488247686 -Enid,49,false,162.074604287669,0043-1234,3.7767032840851327 -Bob,40,true,180.6973555225904,+1 1234,4.976864922452478 -Dylan,43,true,173.8940011972604,+1 1234,3.840024105102519 -Enid,35,false,170.53003190168505,0043-1234,3.146912731530201 -Charles,45,false,175.9191299427927,0043-1234,0.13449729759980267 -Francis,37,false,178.81323593962227,+1 1234,1.8046065918879781 -Francis,37,true,171.64751648501152,0043-1234,1.2121177903237317 -Dylan,21,true,174.23500345515865,0043-1234,2.40193633389855 -Enid,23,true,166.64177130219286,0043-1234,1.3485286041527245 -Enid,28,true,167.84237767602943,+1 1234,3.5604680454430335 -Anne,47,false,173.3829667919652,0043-1234,3.4355520047255492 -Charles,42,true,172.32056905611879,0043-1234,4.572668974525715 -Francis,26,true,153.87122280420334,0043-1234,1.1098493391586837 -Bob,38,false,154.80156174298156,+1 1234,1.09022613940166 -Charles,30,true,175.78649302190107,+1 1234,2.8155094550286215 -Anne,24,false,173.6683115337848,0043-1234,3.411235438552893 -Anne,47,true,164.74234862130373,+1 1234,4.395743541956674 -Charles,39,true,160.344588323495,0043-1234,4.617831663085374 -Anne,41,false,154.21935376553893,0043-1234,3.8765168318277285 -Enid,38,true,161.36130856960816,0043-1234,4.292390362269066 -Dylan,22,true,151.97403458363888,0043-1234,1.511052542912881 -Charles,30,true,160.8150715590866,+1 1234,3.4227094005312617 -Francis,36,false,153.92000405422075,0043-1234,2.57570905818981 -Francis,49,true,162.03322009490762,+1 1234,1.3991670356908592 -Enid,39,false,180.7453190324777,0043-1234,4.170054416280345 -Dylan,30,true,157.385569900398,0043-1234,4.456103378715245 -Enid,24,false,151.93522138656664,0043-1234,3.456877765620685 -Charles,31,false,178.33999739259212,+1 1234,4.708568405839408 -Anne,50,false,151.1441208606709,0043-1234,3.0190218516256966 -Anne,20,false,180.56604162487267,+1 1234,2.47731278312586 +Anne,46,true,150.76531309180402,+1 1234,3.954135362601339 +Charles,38,true,175.04914788698258,+1 1234,4.469747450415067 +Charles,25,false,163.67526465841917,0043-1234,3.8173576150877757 +Dylan,43,true,173.38795478611095,+1 1234,0.8386517266869886 +Dylan,37,false,150.39594757953782,+1 1234,3.1870721894508307 +Francis,24,false,158.31265441139837,0043-1234,2.1981097634077855 +Anne,27,false,173.1515407954204,+1 1234,4.136194212596778 +Enid,25,true,156.05074789133388,0043-1234,1.739556269758793 +Anne,39,false,173.52810777068447,+1 1234,1.2489995417268334 +Bob,50,true,153.42886164781913,0043-1234,1.6770132822982462 +Dylan,21,false,179.2522828395681,0043-1234,1.7863819346480414 +Francis,25,true,156.6346853115135,+1 1234,2.042463741009344 +Francis,24,false,158.11615605341967,+1 1234,2.56949178170946 +Dylan,27,true,161.37139485917572,+1 1234,3.9350862773025206 +Francis,31,true,165.57017518941487,0043-1234,3.814675283289935 +Dylan,30,true,162.62835785785694,0043-1234,1.945897387904975 +Dylan,26,false,159.87970172273054,+1 1234,0.06805021607044015 +Anne,45,true,175.94812549101817,+1 1234,1.3893314815728488 +Francis,45,false,175.7362293138813,+1 1234,3.6052931263877808 +Dylan,25,false,151.88137752123708,+1 1234,3.963878639699785 +Bob,40,false,164.54547044859015,0043-1234,0.23957683440593947 +Enid,23,false,174.56554093303052,+1 1234,4.775962996531939 +Charles,45,true,156.40378367695502,0043-1234,1.9013840006588252 +Enid,33,true,170.24919530588542,+1 1234,1.1282698533772666 +Anne,37,false,152.72646295134584,+1 1234,1.2308853135739772 +Enid,44,false,174.4722597943871,+1 1234,3.6275563138173528 +Charles,44,false,159.4958402131124,0043-1234,2.979171609131394 +Charles,21,false,164.73123593787506,0043-1234,3.5656992014674875 +Dylan,33,false,160.7614554541655,+1 1234,3.6679323706625686 +Charles,29,false,163.09159399378922,+1 1234,1.435471780331461 +Bob,35,true,156.06401957178096,0043-1234,2.88383356208846 +Bob,41,true,168.16142115469412,0043-1234,3.468979035606192 +Dylan,29,false,167.09248840740057,0043-1234,3.1696719532545625 +Anne,50,true,178.38402073531012,0043-1234,2.627006237420564 +Dylan,48,true,176.86038496203497,0043-1234,2.7679507347468877 +Charles,23,false,153.18866644041594,0043-1234,1.6844789082793343 +Charles,43,true,151.6886410617161,0043-1234,1.5357351860506172 +Francis,36,true,172.8508080010261,0043-1234,4.418771255213038 +Francis,50,true,167.39583868406044,0043-1234,3.106473063710372 +Francis,42,true,150.41165672386768,+1 1234,0.682512280008466 +Bob,30,true,179.692215206849,+1 1234,1.1121766883161823 +Anne,27,true,172.32634194445856,0043-1234,3.747433966041404 +Bob,44,true,155.4087890966363,0043-1234,1.4052260971732657 +Charles,24,true,167.4607557553078,+1 1234,1.543574982993518 +Dylan,49,false,170.0994907937394,0043-1234,2.7186027840732 +Anne,50,false,178.8254364074925,0043-1234,0.2387747111609675 +Charles,35,true,162.43650998904812,+1 1234,2.8968735593242143 +Enid,30,false,179.9972920567251,+1 1234,0.9246804514269347 +Francis,31,true,155.18363047992236,+1 1234,3.688335106934169 +Anne,25,false,178.5376031748252,+1 1234,3.0880433201079747 +Enid,47,false,159.55714531518137,0043-1234,4.246690984462306 +Francis,47,false,175.1155653303355,+1 1234,0.11798932212723323 +Francis,33,true,179.26573349293375,+1 1234,3.855848137714783 +Charles,32,true,153.98426698533015,+1 1234,4.794466640988527 +Dylan,46,false,167.03836425161518,0043-1234,0.5417002172687309 +Enid,50,true,166.0505680866778,+1 1234,3.0468200378682226 +Anne,27,false,150.2505335972245,0043-1234,2.4575653099174475 +Dylan,25,false,168.45101965152557,0043-1234,4.369140397150075 +Bob,25,false,150.54892843547685,+1 1234,2.0302319350719222 +Bob,32,false,167.59021977381644,+1 1234,1.0875949075469828 +Francis,41,true,152.41336989771224,0043-1234,1.939612300707253 +Bob,41,true,156.6373051059371,+1 1234,1.242528735163106 +Bob,26,true,179.31203897319696,0043-1234,1.204587839355171 +Anne,20,true,160.4432321594598,0043-1234,4.421969739726977 +Dylan,29,true,151.3826655779588,+1 1234,0.6245551975271046 +Dylan,40,false,159.50021636017874,+1 1234,2.025967529155807 +Enid,42,true,156.72444072948502,0043-1234,3.236276945307121 +Anne,47,false,152.83596037349938,0043-1234,1.8199008858359313 +Charles,20,false,168.21226973203548,+1 1234,4.08819523637704 +Francis,42,true,173.82800661014744,+1 1234,3.4152399822214154 +Bob,27,true,164.86422656509552,0043-1234,2.9836280063834666 +Enid,24,false,159.02999953729767,+1 1234,4.455013335430294 +Dylan,34,false,176.16376462869587,0043-1234,4.463097416427798 +Charles,30,true,154.92876753370254,+1 1234,3.140079811827325 +Anne,23,true,180.12588607054136,0043-1234,3.965476039495355 +Charles,27,true,157.4682354799875,0043-1234,2.99883960404666 +Bob,27,false,150.5910029282255,+1 1234,0.358624561860837 +Francis,30,true,168.39443886823884,0043-1234,1.4061994327272518 +Bob,21,false,151.4865633808954,+1 1234,1.9569325065368817 +Bob,34,false,178.79387629130554,+1 1234,2.8145025433680253 +Anne,45,false,179.9005520055787,+1 1234,4.544631930148121 +Dylan,25,false,175.45510848195835,+1 1234,0.3493017501715543 +Charles,35,false,172.08599798708048,+1 1234,1.8001291618592858 +Charles,43,true,158.14664029762915,+1 1234,4.407331458066724 +Charles,38,true,150.04329329341388,0043-1234,0.9104851878769733 +Anne,33,false,176.92056351345093,+1 1234,2.180236882500436 +Charles,41,true,159.5134654013966,0043-1234,2.6839747977657162 +Dylan,28,true,177.84257944745139,+1 1234,2.5258626507548168 +Enid,25,true,180.95968401074447,+1 1234,4.513885416532654 +Anne,38,true,175.81483444931558,+1 1234,3.5682470398184436 +Anne,27,true,169.40340812938422,+1 1234,1.1624130820711818 +Dylan,27,false,173.4171445681396,0043-1234,2.0519091319479728 +Francis,28,false,160.84177820755508,+1 1234,1.8893825837047076 +Bob,26,false,161.35782863798133,0043-1234,4.152298079763579 +Bob,34,false,161.29137062655272,0043-1234,3.83678871436004 +Enid,32,true,179.8387575766374,0043-1234,1.0064266063376694 +Dylan,26,true,154.3169568388267,0043-1234,2.11365904576754 +Francis,40,true,173.9416162246519,0043-1234,4.62788887546622 +Anne,25,true,172.42133367379003,+1 1234,2.8577621288636665 +Anne,38,false,156.3816430044022,+1 1234,2.936138661291708 diff --git a/testfiles/with-nulls.csv b/testfiles/with-nulls.csv new file mode 100644 index 0000000..695cf2c --- /dev/null +++ b/testfiles/with-nulls.csv @@ -0,0 +1,6 @@ +col1,col2,col3,col4 +hello,1,0.1,true +,2,0.2,false +bye,,0.3,true +test,3,,false +ciao,4,0.4,