diff --git a/package.json b/package.json index 98180a3..4dfb8f5 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "version": "2.1.7", "type": "module", "scripts": { - "test": "vitest", + "test": "TZ=America/Los_Angeles vitest", "prepublishOnly": "rm -rf dist && tsc && chmod +x dist/bin/*.js && cp -r src/styles src/templates dist/src && cp src/runtime/stdlib/*.css dist/src/runtime/stdlib", "lint": "tsc --noEmit && eslint bin src types", "notebooks": "tsx bin/notebooks.ts", diff --git a/src/runtime/stdlib/dsv.test.ts b/src/runtime/stdlib/dsv.test.ts new file mode 100644 index 0000000..b54fcb0 --- /dev/null +++ b/src/runtime/stdlib/dsv.test.ts @@ -0,0 +1,88 @@ +import {assert, describe, it} from "vitest"; +import {inferTypes} from "./dsv"; +import {coerceBoolean, coerceDate, coerceNumber} from "./dsv"; + +function inferType(values: string[]): unknown[] { + return inferTypes(values.map((value) => ({value})), ["value"]).map(({value}) => value); // prettier-ignore +} + +describe("inferTypes", () => { + it("infers numbers", () => { + assert.deepStrictEqual(inferType(["2", "4", "6"]), [2, 4, 6]); + assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), [1.2, 3.4, 5.67]); + assert.deepStrictEqual(inferType([".2", ".4", ".67"]), [0.2, 0.4, 0.67]); + assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9].map(String)), [0.1, 1, 2, 3, 9]); + assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), [NaN, 1, 2, 3, 9, 1, 1, 1, 3, 2, 3]); // prettier-ignore + assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), [0, 0.1, 2, 3, 4, 5, 2, 3, 0.9, NaN]); // prettier-ignore + }); + it("infers booleans", () => { + assert.deepStrictEqual(inferType(["true", "", "false"]), [true, null, false]); + assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, "", "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, null, undefined]); // prettier-ignore + }); + it("infers dates in common formats", () => { + assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T08:00:00.000Z"), new Date("2020-11-12T20:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore + }); + it("infers strings", () => { + assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), ["cat", "dog", "1,000", "null"]); // prettier-ignore + assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), ["10n", "22n", "0n"]); + assert.deepStrictEqual(inferType(["", ""]), ["", ""]); + assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), ["NaN", "NaN", "NaN", "1", "2", "3"]); // prettier-ignore + assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), ["x", 1, 2, 3, 9, 1, 1, 3].map(String)); // prettier-ignore + assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), [".0", ".1", "2", "3", "4", "5", "2", "3", "x"]); // prettier-ignore + assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), ["true", "false", "true", "false", "true", "false", "pants on fire"]); // prettier-ignore + }); +}); + +describe("coerceNumber", () => { + it("coerces to number", () => { + assert.deepStrictEqual(coerceNumber("1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2); + assert.deepStrictEqual(coerceNumber("10"), 10); + assert.deepStrictEqual(coerceNumber("0"), 0); + assert.deepStrictEqual(coerceNumber("A"), NaN); + assert.deepStrictEqual(coerceNumber(""), NaN); + assert.deepStrictEqual(coerceNumber(" "), NaN); + assert.deepStrictEqual(coerceNumber("null"), NaN); + assert.deepStrictEqual(coerceNumber("1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2); + assert.deepStrictEqual(coerceNumber("0"), 0); + assert.deepStrictEqual(coerceNumber(""), NaN); + assert.deepStrictEqual(coerceNumber(" "), NaN); + assert.deepStrictEqual(coerceNumber("A"), NaN); + assert.deepStrictEqual(coerceNumber("null"), NaN); + assert.deepStrictEqual(coerceNumber("undefined"), NaN); + assert.deepStrictEqual(coerceNumber("{a: 1}"), NaN); + }); +}); + +describe("coerceBoolean", () => { + it("coerces to boolean", () => { + assert.deepStrictEqual(coerceBoolean("true"), true); + assert.deepStrictEqual(coerceBoolean("True "), true); + assert.deepStrictEqual(coerceBoolean("False"), false); + assert.deepStrictEqual(coerceBoolean("false"), false); + assert.deepStrictEqual(coerceBoolean(""), null); + assert.deepStrictEqual(coerceBoolean("1"), undefined); + assert.deepStrictEqual(coerceBoolean("2"), undefined); + assert.deepStrictEqual(coerceBoolean("0"), undefined); + assert.deepStrictEqual(coerceBoolean("null"), undefined); + assert.deepStrictEqual(coerceBoolean("undefined"), undefined); + }); +}); + +describe("coerceDate", () => { + it("coerces to date", () => { + const invalidDate = new Date(NaN); + assert.deepStrictEqual(coerceDate("12/12/2020"), new Date("12/12/2020")); + assert.deepStrictEqual(coerceDate("12/12/2020 "), new Date("12/12/2020")); + assert.deepStrictEqual(coerceDate("2022-01-01T12:34:00Z"), new Date("2022-01-01T12:34:00Z")); // prettier-ignore + assert.deepStrictEqual(coerceDate("{a: 1}"), invalidDate); + assert.deepStrictEqual(coerceDate("true"), invalidDate); + assert.deepStrictEqual(coerceDate("2020-1-12"), invalidDate); + assert.deepStrictEqual(coerceDate("1675356739000"), invalidDate); + assert.deepStrictEqual(coerceDate("undefined"), invalidDate); + assert.deepStrictEqual(coerceDate(""), null); + }); +}); diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts new file mode 100644 index 0000000..889cec1 --- /dev/null +++ b/src/runtime/stdlib/dsv.ts @@ -0,0 +1,75 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions */ + +/** + * Accepts dates in the following forms: + * - ±YYYYYY-MM-DD + * - YYYY-MM-DD + * - MM/DD/YY + * - MM/DD/YYYY + * + * Following a "T" or a space, dates may additionally have a time: + * - HH:MM + * - HH:MM:SS + * - HH:MM:SS.MMM + * + * And the time may optionally have a time zone: + * - Z + * - ±HH:MM + */ +const DATE_TEST = /^([-+]\d{2})?\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{2,4}([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore + +/** The maximum number of rows to sample (including missing values). */ +const SAMPLE_SIZE = 100; + +export function inferTypes>( + rows: T[], + columns: (keyof T)[] +): Record[] { + const output = rows as Record[]; + const n = rows.length; + const k = Math.min(n, SAMPLE_SIZE); + for (const column of columns) { + let booleans = 0; + let numbers = 0; + let dates = 0; + let strings = 0; + + for (let i = 0; i < k; ++i) { + const value = rows[i][column]?.trim(); + if (!value) continue; + ++strings; + if (/^(true|false)$/i.test(value)) ++booleans; + else if (!isNaN(Number(value))) ++numbers; + else if (DATE_TEST.test(value)) ++dates; + } + + // Chose the non-string type with the greatest count that is also ≥90%; or + // if no such type meets that criterion, use string. + let coerce: ((value: string) => unknown) | undefined = undefined; + let typeCount = Math.max(1, Math.ceil(strings * 0.9)) - 1; + if (booleans > typeCount) ((coerce = coerceBoolean), (typeCount = booleans)); + if (numbers > typeCount) ((coerce = coerceNumber), (typeCount = numbers)); + if (dates > typeCount) ((coerce = coerceDate), (typeCount = dates)); + if (!coerce) continue; + + for (let i = 0; i < n; ++i) { + output[i][column] = coerce(rows[i][column]); + } + } + return output; +} + +export function coerceBoolean(value: string): boolean | null | undefined { + const trimmed = value.trim().toLowerCase(); + return trimmed === "true" ? true : trimmed === "false" ? false : trimmed ? undefined : null; +} + +export function coerceNumber(value: string): number { + const trimmed = value.trim(); + return trimmed ? Number(value) : NaN; +} + +export function coerceDate(value: string): Date | null { + const trimmed = value.trim(); + return trimmed ? new Date(DATE_TEST.test(trimmed) ? trimmed : NaN) : null; +} diff --git a/src/runtime/stdlib/fileAttachment.ts b/src/runtime/stdlib/fileAttachment.ts index b448646..cbc868b 100644 --- a/src/runtime/stdlib/fileAttachment.ts +++ b/src/runtime/stdlib/fileAttachment.ts @@ -1,7 +1,9 @@ +import {inferTypes} from "./dsv.js"; + /* eslint-disable @typescript-eslint/no-explicit-any */ const files = new Map(); -export type DsvOptions = {delimiter?: string; array?: boolean; typed?: boolean}; +export type DsvOptions = {delimiter?: string; array?: boolean; typed?: "auto" | boolean}; export type DsvResult = (Record[] | any[][]) & {columns: string[]}; export interface FileAttachment { @@ -136,11 +138,13 @@ export abstract class AbstractFile implements FileAttachment { async stream(): Promise>> { return (await fetchFile(this)).body!; } - async dsv({delimiter = ",", array = false, typed = false} = {}): Promise { + async dsv({delimiter = ",", array = false, typed = false}: DsvOptions = {}): Promise { const [text, d3] = await Promise.all([this.text(), import("https://cdn.jsdelivr.net/npm/d3-dsv/+esm")]); // prettier-ignore const format = d3.dsvFormat(delimiter); - const parse = array ? format.parseRows : format.parse; - return parse(text, typed && d3.autoType); + const parse = array ? ((typed &&= true), format.parseRows) : format.parse; + const output = parse(text, typed === true ? d3.autoType : undefined); + if (typed === "auto") inferTypes(output, output.columns); + return output; } async csv(options?: Omit): Promise { return this.dsv({...options, delimiter: ","});