From 291b116509308a6c2a9f86572daee3249f8b72f3 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Sat, 13 Jun 2026 19:58:19 -0700 Subject: [PATCH 1/9] inferSchema, enforceSchema --- src/runtime/stdlib/dsv.ts | 185 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 src/runtime/stdlib/dsv.ts diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts new file mode 100644 index 0000000..425de7c --- /dev/null +++ b/src/runtime/stdlib/dsv.ts @@ -0,0 +1,185 @@ +import type {ColumnSchema} from "./databaseClient.js"; + +/** Accepts dates in the form of ISOString and LocaleDateString, with or without time. */ +const DATE_TEST = /^(([-+]\d{2})?\d{4}(-\d{2}(-\d{2}))|(\d{1,2})\/(\d{1,2})\/(\d{2,4}))([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore + +export function inferSchema( + source: Record[], + columns = getAllKeys(source) +): ColumnSchema[] { + const schema: ColumnSchema[] = []; + const sampleSize = 100; + const sample = source.slice(0, sampleSize); + const typeCounts: Record = {}; + for (const col of columns) { + const colCount = (typeCounts[col] = createTypeCount()); + for (const d of sample) { + let value = d[col]; + if (value == null) continue; + // const type = typeof value; + // if (type !== "string") { + // ++colCount.defined; + // if (Array.isArray(value)) ++colCount.array; + // else if (value instanceof Date) ++colCount.date; + // else if (value instanceof ArrayBuffer) ++colCount.buffer; + // else if (type === "number") { + // ++colCount.number; + // if (Number.isInteger(value)) ++colCount.integer; + // } + // // bigint, boolean, or object + // else if (type in colCount) ++colCount[type]; + // } else { + value = value.trim(); + if (!value) continue; + ++colCount.defined; + ++colCount.string; + if (/^(true|false)$/i.test(value)) { + ++colCount.boolean; + } else if (value && !isNaN(Number(value))) { + ++colCount.number; + if (Number.isInteger(+value)) ++colCount.integer; + } else if (DATE_TEST.test(value)) ++colCount.date; + // } + } + // Chose the non-string, non-other type with the greatest count that is also + // ≥90%; or if no such type meets that criterion, fall back to string if + // ≥90%; and lastly fall back to other. + const minCount = Math.max(1, colCount.defined * 0.9); + let type: Type = "other"; + let typeCount = minCount - 1; + for (const inferrableType of inferrableTypes) { + if (colCount[inferrableType] > typeCount) { + type = inferrableType; + typeCount = colCount[type]; + } + } + if (colCount.string > typeCount) { + type = "string"; + typeCount = colCount.string; + } + schema.push({name: col, type}); + } + return schema; +} + +export function enforceSchema( + source: Record[], + schema: ColumnSchema[] +): Record[] & {schema: typeof schema} { + const types = new Map(schema.map(({name, type}) => [name, type])); + return Object.assign( + source.map((d) => coerceRow(d, types, schema)), + {schema} + ); +} + +function coerceRow( + object: Record, + types: Map, + schema: ColumnSchema[] +) { + const coerced: Record = {}; + for (const col of schema) { + const type = types.get(col.name)!; + const value = object[col.name]; + coerced[col.name] = type === "raw" ? value : coerceToType(value, type); + } + return coerced; +} + +function coerceToType(value: unknown, type: Type): unknown { + switch (type) { + case "string": + return typeof value === "string" || value == null ? value : String(value); + case "boolean": + if (typeof value === "string") { + const trimValue = value.trim().toLowerCase(); + return trimValue === "true" ? true : trimValue === "false" ? false : null; + } + return typeof value === "boolean" || value == null ? value : Boolean(value); + case "bigint": + return typeof value === "bigint" || value == null + ? value + : Number.isInteger(typeof value === "string" && !value.trim() ? NaN : +value) + ? BigInt(value as string) + : undefined; + case "integer": // not a target type for coercion, but can be inferred + case "number": { + return typeof value === "number" + ? value + : value == null || (typeof value === "string" && !value.trim()) + ? NaN + : Number(value); + } + case "date": { + if (value instanceof Date || value == null) return value; + if (typeof value === "number") return new Date(value); + const trimValue = String(value).trim(); + if (typeof value === "string" && !trimValue) return null; + return new Date(DATE_TEST.test(trimValue) ? trimValue : NaN); + } + case "array": + case "object": + case "buffer": + case "other": + return value; + default: + throw new Error(`Unable to coerce to type: ${type}`); + } +} + +function createTypeCount(): TypeCount { + return { + boolean: 0, + integer: 0, + number: 0, + date: 0, + string: 0, + array: 0, + object: 0, + bigint: 0, + buffer: 0, + defined: 0 + }; +} + +type TypeCount = Record | "defined", number>; + +type Type = + | "boolean" + | "integer" + | "number" + | "date" + | "bigint" + | "array" + | "object" + | "buffer" + | "string" + | "other"; + +// Caution: the order below matters! 🌶️ The first one that passes the ≥90% test +// should be the one that we chose, and therefore these types should be listed +// from most specific to least specific. +const inferrableTypes: Exclude[] = [ + "boolean", + "integer", + "number", + "date", + "bigint", + "array", + "object", + "buffer" + // Note: "other" and "string" are intentionally omitted; see below! +]; + +function getAllKeys(rows: Record[]): string[] { + const keys = new Set(); + for (const row of rows) { + if (row != null) { + for (const key of Object.keys(row)) { + keys.add(key); + } + } + } + return Array.from(keys); +} From 7c39438944cf8178ad7b227f6feba0a25b02d32d Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Sun, 14 Jun 2026 19:02:13 -0700 Subject: [PATCH 2/9] checkpoint tests --- src/runtime/stdlib/dsv.test.ts | 247 +++++++++++++++++++++++++++++++++ src/runtime/stdlib/dsv.ts | 30 ++-- 2 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 src/runtime/stdlib/dsv.test.ts diff --git a/src/runtime/stdlib/dsv.test.ts b/src/runtime/stdlib/dsv.test.ts new file mode 100644 index 0000000..cf5e8bd --- /dev/null +++ b/src/runtime/stdlib/dsv.test.ts @@ -0,0 +1,247 @@ +import {assert, describe, it} from "vitest"; +import type {Type} from "./dsv"; +import {coerceToType, enforceSchema, inferSchema} from "./dsv"; + +describe("inferSchema", () => { + function inferType(values: string[]): Type { + return inferSchema(values.map((value) => ({value})))[0].type; + } + + it("infers the union of columns", () => { + assert.deepStrictEqual(inferSchema([{a: "2"}, {b: "bee"}, {c: "true"}]), [ + {name: "a", type: "integer"}, + {name: "b", type: "string"}, + {name: "c", type: "boolean"} + ]); + }); + + it("infers integers", () => { + assert.deepStrictEqual(inferType(["2", "4", "6"]), "integer"); + }); + + it("infers numbers", () => { + assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), "number"); + }); + + it("infers numbers with no leading zero", () => { + assert.deepStrictEqual(inferType([".2", ".4", ".67"]), "number"); + }); + + it("infers booleans", () => { + assert.deepStrictEqual(inferType(["true", "", "false"]), "boolean"); + }); + + it("infers dates in common formats", () => { + assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), "date"); + }); + + it("infers strings", () => { + assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), "string"); + }); + + it('considers "42n" to be a string, not a bigint', () => { + assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), "string"); + }); + + it("considers (exclusively) empty strings to be strings", () => { + assert.deepStrictEqual(inferType(["", ""]), "string"); + }); + + it("considers mixed integers and numbers to be numbers, not integers", () => { + assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9, 10, 11, 12, 13].map(String)), "number"); + }); + + it('considers "NaN" to be a string, not a number', () => { + assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), "string"); + }); + + it("infers mostly integers as integers", () => { + assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), "integer"); + }); + + it("infers partly integers as strings", () => { + assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), "string"); + }); + + it("infers mostly numbers as numbers", () => { + assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), "number"); // prettier-ignore + }); + + it("infers partly numbers as strings", () => { + assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), "string"); + }); + + it("infers mostly booleans as booleans", () => { + assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, false, "pants on fire"].map(String)), "boolean"); // prettier-ignore + }); + + it("infers partly booleans as strings", () => { + assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), "string"); // prettier-ignore + }); +}); + +describe("enforceSchema", () => { + it.skip("coerces an array of objects", () => { + const source = [{a: "0", b: "1", c: "2"}]; + assert.deepStrictEqual( + enforceSchema(source, inferSchema(source)), + Object.assign([{a: 0, b: 1, c: 2}], { + schema: [ + { + name: "a", + type: "integer" + }, + { + name: "b", + type: "integer" + }, + { + name: "c", + type: "integer" + } + ] + }) + ); + }); +}); + +describe("coerceToType", () => { + it.skip("coerces to integer", () => { + // "integer" is not a target type for coercion, but can be inferred. So it + // will be handled as an alias for "number". + assert.deepStrictEqual(coerceToType("1.2", "integer"), 1.2); + assert.deepStrictEqual(coerceToType(" 1.2", "integer"), 1.2); + assert.deepStrictEqual(coerceToType(" 1.2 ", "integer"), 1.2); + assert.deepStrictEqual(coerceToType(1.2, "integer"), 1.2); + assert.deepStrictEqual(coerceToType("10", "integer"), 10); + assert.deepStrictEqual(coerceToType(0, "integer"), 0); + assert.deepStrictEqual(coerceToType("A", "integer"), NaN); + assert.deepStrictEqual(coerceToType("", "integer"), NaN); + assert.deepStrictEqual(coerceToType(" ", "integer"), NaN); + assert.deepStrictEqual(coerceToType(null, "integer"), NaN); + }); + + it.skip("coerces to number", () => { + assert.deepStrictEqual(coerceToType("1.2", "number"), 1.2); + assert.deepStrictEqual(coerceToType(" 1.2", "number"), 1.2); + assert.deepStrictEqual(coerceToType(" 1.2 ", "number"), 1.2); + assert.deepStrictEqual(coerceToType(0, "number"), 0); + assert.deepStrictEqual(coerceToType("", "number"), NaN); + assert.deepStrictEqual(coerceToType(" ", "number"), NaN); + assert.deepStrictEqual(coerceToType("A", "number"), NaN); + assert.deepStrictEqual(coerceToType(null, "number"), NaN); + assert.deepStrictEqual(coerceToType(undefined, "number"), NaN); + assert.deepStrictEqual(coerceToType({a: 1}, "number"), NaN); + }); + + it.skip("coerces to boolean", () => { + assert.deepStrictEqual(coerceToType("true", "boolean"), true); + assert.deepStrictEqual(coerceToType("True ", "boolean"), true); + assert.deepStrictEqual(coerceToType(true, "boolean"), true); + assert.deepStrictEqual(coerceToType("False", "boolean"), false); + assert.deepStrictEqual(coerceToType(false, "boolean"), false); + assert.deepStrictEqual(coerceToType(1, "boolean"), true); + assert.deepStrictEqual(coerceToType(2, "boolean"), true); + assert.deepStrictEqual(coerceToType(0, "boolean"), false); + assert.deepStrictEqual(coerceToType({}, "boolean"), true); + assert.deepStrictEqual(coerceToType(new Date(), "boolean"), true); + assert.deepStrictEqual(coerceToType("A", "boolean"), null); + assert.deepStrictEqual(coerceToType("", "boolean"), null); + assert.deepStrictEqual(coerceToType(" ", "boolean"), null); + assert.deepStrictEqual(coerceToType(null, "boolean"), null); + assert.deepStrictEqual(coerceToType(undefined, "boolean"), undefined); + }); + + it.skip("coerces to date", () => { + const invalidDate = new Date(NaN); + assert.deepStrictEqual(coerceToType("12/12/2020", "date"), new Date("12/12/2020")); + // with whitespace + assert.deepStrictEqual(coerceToType("12/12/2020 ", "date"), new Date("12/12/2020")); + assert.deepStrictEqual( + coerceToType("2022-01-01T12:34:00Z", "date"), + new Date("2022-01-01T12:34:00Z") + ); + assert.deepStrictEqual(coerceToType({a: 1}, "date").toString(), invalidDate.toString()); + assert.deepStrictEqual(coerceToType(true, "date").toString(), invalidDate.toString()); + assert.deepStrictEqual(coerceToType("2020-1-12", "date").toString(), invalidDate.toString()); + assert.deepStrictEqual(coerceToType(1675356739000, "date"), new Date(1675356739000)); + assert.deepStrictEqual(coerceToType(undefined, "date"), undefined); + assert.deepStrictEqual(coerceToType(null, "date"), null); + assert.deepStrictEqual(coerceToType("", "date"), null); + assert.deepStrictEqual(coerceToType(" ", "date"), null); + assert.deepStrictEqual( + coerceToType({toString: () => " "}, "date").toString(), + invalidDate.toString() + ); + assert.deepStrictEqual( + coerceToType({toString: () => "2020-01-01"}, "date"), + new Date("2020-01-01") + ); + }); + + it.skip("coerces to string", () => { + assert.deepStrictEqual(coerceToType(true, "string"), "true"); + assert.deepStrictEqual(coerceToType(false, "string"), "false"); + assert.deepStrictEqual(coerceToType(10, "string"), "10"); + assert.deepStrictEqual(coerceToType({a: 1}, "string"), "[object Object]"); + assert.deepStrictEqual(coerceToType(0, "string"), "0"); + assert.deepStrictEqual(coerceToType("", "string"), ""); + assert.deepStrictEqual(coerceToType(" ", "string"), " "); + assert.deepStrictEqual(coerceToType(" foo", "string"), " foo"); + assert.deepStrictEqual(coerceToType(" foo ", "string"), " foo "); + assert.deepStrictEqual(coerceToType(null, "string"), null); + assert.deepStrictEqual(coerceToType(undefined, "string"), undefined); + assert.deepStrictEqual(coerceToType(NaN, "string"), "NaN"); + }); + + it.skip("coerces to bigint", () => { + assert.deepStrictEqual(coerceToType("32", "bigint"), 32n); + assert.deepStrictEqual(coerceToType(" 32", "bigint"), 32n); + assert.deepStrictEqual(coerceToType(32n, "bigint"), 32n); + assert.deepStrictEqual(coerceToType(0, "bigint"), 0n); + assert.deepStrictEqual(coerceToType(false, "bigint"), 0n); + assert.deepStrictEqual(coerceToType(true, "bigint"), 1n); + assert.deepStrictEqual(coerceToType(null, "bigint"), null); + assert.deepStrictEqual(coerceToType(undefined, "bigint"), undefined); + assert.deepStrictEqual(coerceToType(1.1, "bigint"), undefined); + assert.deepStrictEqual(coerceToType("1.1", "bigint"), undefined); + assert.deepStrictEqual(coerceToType(" 32n", "bigint"), undefined); + assert.deepStrictEqual(coerceToType("A", "bigint"), undefined); + assert.deepStrictEqual(coerceToType("", "bigint"), undefined); + assert.deepStrictEqual(coerceToType(" ", "bigint"), undefined); + assert.deepStrictEqual(coerceToType(NaN, "bigint"), undefined); + }); + + it.skip("coerces to array", () => { + // "array" is not a target type for coercion, but can be inferred. + assert.deepStrictEqual(coerceToType([1, 2, 3], "array"), [1, 2, 3]); + assert.deepStrictEqual(coerceToType(null, "array"), null); + assert.deepStrictEqual(coerceToType(undefined, "array"), undefined); + }); + + it.skip("coerces to object", () => { + // "object" is not a target type for coercion, but can be inferred. + assert.deepStrictEqual(coerceToType({a: 1, b: 2}, "object"), {a: 1, b: 2}); + assert.deepStrictEqual(coerceToType(null, "object"), null); + assert.deepStrictEqual(coerceToType(undefined, "object"), undefined); + }); + + it.skip("coerces to buffer", () => { + // "buffer" is not a target type for coercion, but can be inferred. + assert.deepStrictEqual(coerceToType(new ArrayBuffer(), "buffer"), new ArrayBuffer()); + assert.deepStrictEqual(coerceToType("A", "buffer"), "A"); + assert.deepStrictEqual(coerceToType(null, "buffer"), null); + assert.deepStrictEqual(coerceToType(undefined, "buffer"), undefined); + }); + + it.skip("coerces to other", () => { + // "other" is not a target type for coercion, but can be inferred. + assert.deepStrictEqual(coerceToType(0, "other"), 0); + assert.deepStrictEqual(coerceToType("a", "other"), "a"); + assert.deepStrictEqual(coerceToType(null, "other"), null); + assert.deepStrictEqual(coerceToType(undefined, "other"), undefined); + }); + + // Note: if type is "raw", coerceToType() will not be called. Instead, values + // will be returned from coerceRow(). +}); diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts index 425de7c..ef024c9 100644 --- a/src/runtime/stdlib/dsv.ts +++ b/src/runtime/stdlib/dsv.ts @@ -44,19 +44,20 @@ export function inferSchema( // Chose the non-string, non-other type with the greatest count that is also // ≥90%; or if no such type meets that criterion, fall back to string if // ≥90%; and lastly fall back to other. - const minCount = Math.max(1, colCount.defined * 0.9); - let type: Type = "other"; + const minCount = Math.max(1, Math.ceil(colCount.defined * 0.9)); + let type: Type = "string"; let typeCount = minCount - 1; + console.warn(colCount, typeCount) for (const inferrableType of inferrableTypes) { if (colCount[inferrableType] > typeCount) { type = inferrableType; typeCount = colCount[type]; } } - if (colCount.string > typeCount) { - type = "string"; - typeCount = colCount.string; - } + // if (colCount.string > typeCount) { + // type = "string"; + // typeCount = colCount.string; + // } schema.push({name: col, type}); } return schema; @@ -73,7 +74,7 @@ export function enforceSchema( ); } -function coerceRow( +export function coerceRow( object: Record, types: Map, schema: ColumnSchema[] @@ -87,7 +88,7 @@ function coerceRow( return coerced; } -function coerceToType(value: unknown, type: Type): unknown { +export function coerceToType(value: unknown, type: Type): unknown { switch (type) { case "string": return typeof value === "string" || value == null ? value : String(value); @@ -145,7 +146,7 @@ function createTypeCount(): TypeCount { type TypeCount = Record | "defined", number>; -type Type = +export type Type = | "boolean" | "integer" | "number" @@ -165,14 +166,15 @@ const inferrableTypes: Exclude[] = [ "integer", "number", "date", - "bigint", - "array", - "object", - "buffer" + // "string" + // "bigint", + // "array", + // "object", + // "buffer" // Note: "other" and "string" are intentionally omitted; see below! ]; -function getAllKeys(rows: Record[]): string[] { +export function getAllKeys(rows: Record[]): string[] { const keys = new Set(); for (const row of rows) { if (row != null) { From b3d77cc800161bb8119e137d0bcee75a04ea2eee Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 15 Jun 2026 19:48:09 -0600 Subject: [PATCH 3/9] rewrite everything --- src/runtime/stdlib/dsv.test.ts | 248 ++++++++++----------------------- src/runtime/stdlib/dsv.ts | 205 +++++++++------------------ 2 files changed, 133 insertions(+), 320 deletions(-) diff --git a/src/runtime/stdlib/dsv.test.ts b/src/runtime/stdlib/dsv.test.ts index cf5e8bd..89368de 100644 --- a/src/runtime/stdlib/dsv.test.ts +++ b/src/runtime/stdlib/dsv.test.ts @@ -1,12 +1,16 @@ import {assert, describe, it} from "vitest"; import type {Type} from "./dsv"; -import {coerceToType, enforceSchema, inferSchema} from "./dsv"; +import {coerceValue, enforceSchema, inferSchema} from "./dsv"; -describe("inferSchema", () => { - function inferType(values: string[]): Type { - return inferSchema(values.map((value) => ({value})))[0].type; - } +function inferType(values: string[]): Type { + return inferSchema(values.map((value) => ({value})))[0].type; +} + +function enforceType(values: string[], type: Type): unknown[] { + return enforceSchema(values.map((value) => ({value})), [{name: "value", type}]).map(({value}) => value); // prettier-ignore +} +describe("inferSchema", () => { it("infers the union of columns", () => { assert.deepStrictEqual(inferSchema([{a: "2"}, {b: "bee"}, {c: "true"}]), [ {name: "a", type: "integer"}, @@ -14,234 +18,124 @@ describe("inferSchema", () => { {name: "c", type: "boolean"} ]); }); - it("infers integers", () => { assert.deepStrictEqual(inferType(["2", "4", "6"]), "integer"); }); - it("infers numbers", () => { assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), "number"); }); - it("infers numbers with no leading zero", () => { assert.deepStrictEqual(inferType([".2", ".4", ".67"]), "number"); }); - it("infers booleans", () => { assert.deepStrictEqual(inferType(["true", "", "false"]), "boolean"); }); - it("infers dates in common formats", () => { assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), "date"); }); - it("infers strings", () => { assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), "string"); }); - it('considers "42n" to be a string, not a bigint', () => { assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), "string"); }); - it("considers (exclusively) empty strings to be strings", () => { assert.deepStrictEqual(inferType(["", ""]), "string"); }); - it("considers mixed integers and numbers to be numbers, not integers", () => { assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9, 10, 11, 12, 13].map(String)), "number"); }); - it('considers "NaN" to be a string, not a number', () => { assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), "string"); }); - it("infers mostly integers as integers", () => { assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), "integer"); }); - it("infers partly integers as strings", () => { assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), "string"); }); - it("infers mostly numbers as numbers", () => { assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), "number"); // prettier-ignore }); - it("infers partly numbers as strings", () => { assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), "string"); }); - it("infers mostly booleans as booleans", () => { assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, false, "pants on fire"].map(String)), "boolean"); // prettier-ignore }); - it("infers partly booleans as strings", () => { assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), "string"); // prettier-ignore }); }); describe("enforceSchema", () => { - it.skip("coerces an array of objects", () => { - const source = [{a: "0", b: "1", c: "2"}]; - assert.deepStrictEqual( - enforceSchema(source, inferSchema(source)), - Object.assign([{a: 0, b: 1, c: 2}], { - schema: [ - { - name: "a", - type: "integer" - }, - { - name: "b", - type: "integer" - }, - { - name: "c", - type: "integer" - } - ] - }) - ); + it("enforces integers", () => { + assert.deepStrictEqual(enforceType(["1", "2", "4", "3", ""], "integer"), [1, 2, 4, 3, NaN]); }); -}); - -describe("coerceToType", () => { - it.skip("coerces to integer", () => { - // "integer" is not a target type for coercion, but can be inferred. So it - // will be handled as an alias for "number". - assert.deepStrictEqual(coerceToType("1.2", "integer"), 1.2); - assert.deepStrictEqual(coerceToType(" 1.2", "integer"), 1.2); - assert.deepStrictEqual(coerceToType(" 1.2 ", "integer"), 1.2); - assert.deepStrictEqual(coerceToType(1.2, "integer"), 1.2); - assert.deepStrictEqual(coerceToType("10", "integer"), 10); - assert.deepStrictEqual(coerceToType(0, "integer"), 0); - assert.deepStrictEqual(coerceToType("A", "integer"), NaN); - assert.deepStrictEqual(coerceToType("", "integer"), NaN); - assert.deepStrictEqual(coerceToType(" ", "integer"), NaN); - assert.deepStrictEqual(coerceToType(null, "integer"), NaN); + it("enforces numbers", () => { + assert.deepStrictEqual(enforceType(["1.2", "4.3", "0.1", ""], "number"), [1.2, 4.3, 0.1, NaN]); }); - - it.skip("coerces to number", () => { - assert.deepStrictEqual(coerceToType("1.2", "number"), 1.2); - assert.deepStrictEqual(coerceToType(" 1.2", "number"), 1.2); - assert.deepStrictEqual(coerceToType(" 1.2 ", "number"), 1.2); - assert.deepStrictEqual(coerceToType(0, "number"), 0); - assert.deepStrictEqual(coerceToType("", "number"), NaN); - assert.deepStrictEqual(coerceToType(" ", "number"), NaN); - assert.deepStrictEqual(coerceToType("A", "number"), NaN); - assert.deepStrictEqual(coerceToType(null, "number"), NaN); - assert.deepStrictEqual(coerceToType(undefined, "number"), NaN); - assert.deepStrictEqual(coerceToType({a: 1}, "number"), NaN); + it("enforces booleans", () => { + assert.deepStrictEqual(enforceType(["true", "false", ""], "boolean"), [true, false, null]); }); - - it.skip("coerces to boolean", () => { - assert.deepStrictEqual(coerceToType("true", "boolean"), true); - assert.deepStrictEqual(coerceToType("True ", "boolean"), true); - assert.deepStrictEqual(coerceToType(true, "boolean"), true); - assert.deepStrictEqual(coerceToType("False", "boolean"), false); - assert.deepStrictEqual(coerceToType(false, "boolean"), false); - assert.deepStrictEqual(coerceToType(1, "boolean"), true); - assert.deepStrictEqual(coerceToType(2, "boolean"), true); - assert.deepStrictEqual(coerceToType(0, "boolean"), false); - assert.deepStrictEqual(coerceToType({}, "boolean"), true); - assert.deepStrictEqual(coerceToType(new Date(), "boolean"), true); - assert.deepStrictEqual(coerceToType("A", "boolean"), null); - assert.deepStrictEqual(coerceToType("", "boolean"), null); - assert.deepStrictEqual(coerceToType(" ", "boolean"), null); - assert.deepStrictEqual(coerceToType(null, "boolean"), null); - assert.deepStrictEqual(coerceToType(undefined, "boolean"), undefined); + it("enforces dates", () => { + assert.deepStrictEqual(enforceType(["2002-01-01", "2003-01-01", ""], "date"), [new Date("2002-01-01"), new Date("2003-01-01"), null]); // prettier-ignore }); +}); - it.skip("coerces to date", () => { +describe("coerceValue", () => { + it("coerces to integer", () => { + assert.deepStrictEqual(coerceValue("1.2", "integer"), 1.2); + assert.deepStrictEqual(coerceValue(" 1.2", "integer"), 1.2); + assert.deepStrictEqual(coerceValue(" 1.2 ", "integer"), 1.2); + assert.deepStrictEqual(coerceValue("10", "integer"), 10); + assert.deepStrictEqual(coerceValue("0", "integer"), 0); + assert.deepStrictEqual(coerceValue("A", "integer"), NaN); + assert.deepStrictEqual(coerceValue("", "integer"), NaN); + assert.deepStrictEqual(coerceValue(" ", "integer"), NaN); + assert.deepStrictEqual(coerceValue("null", "integer"), NaN); + }); + it("coerces to number", () => { + assert.deepStrictEqual(coerceValue("1.2", "number"), 1.2); + assert.deepStrictEqual(coerceValue(" 1.2", "number"), 1.2); + assert.deepStrictEqual(coerceValue(" 1.2 ", "number"), 1.2); + assert.deepStrictEqual(coerceValue("0", "number"), 0); + assert.deepStrictEqual(coerceValue("", "number"), NaN); + assert.deepStrictEqual(coerceValue(" ", "number"), NaN); + assert.deepStrictEqual(coerceValue("A", "number"), NaN); + assert.deepStrictEqual(coerceValue("null", "number"), NaN); + assert.deepStrictEqual(coerceValue("undefined", "number"), NaN); + assert.deepStrictEqual(coerceValue("{a: 1}", "number"), NaN); + }); + it("coerces to boolean", () => { + assert.deepStrictEqual(coerceValue("true", "boolean"), true); + assert.deepStrictEqual(coerceValue("True ", "boolean"), true); + assert.deepStrictEqual(coerceValue("False", "boolean"), false); + assert.deepStrictEqual(coerceValue("false", "boolean"), false); + assert.deepStrictEqual(coerceValue("1", "boolean"), null); // TODO null vs. undefined? + assert.deepStrictEqual(coerceValue("2", "boolean"), null); + assert.deepStrictEqual(coerceValue("0", "boolean"), null); + assert.deepStrictEqual(coerceValue("{}", "boolean"), null); + assert.deepStrictEqual(coerceValue("null", "boolean"), null); + assert.deepStrictEqual(coerceValue("undefined", "boolean"), null); + }); + it("coerces to date", () => { const invalidDate = new Date(NaN); - assert.deepStrictEqual(coerceToType("12/12/2020", "date"), new Date("12/12/2020")); - // with whitespace - assert.deepStrictEqual(coerceToType("12/12/2020 ", "date"), new Date("12/12/2020")); - assert.deepStrictEqual( - coerceToType("2022-01-01T12:34:00Z", "date"), - new Date("2022-01-01T12:34:00Z") - ); - assert.deepStrictEqual(coerceToType({a: 1}, "date").toString(), invalidDate.toString()); - assert.deepStrictEqual(coerceToType(true, "date").toString(), invalidDate.toString()); - assert.deepStrictEqual(coerceToType("2020-1-12", "date").toString(), invalidDate.toString()); - assert.deepStrictEqual(coerceToType(1675356739000, "date"), new Date(1675356739000)); - assert.deepStrictEqual(coerceToType(undefined, "date"), undefined); - assert.deepStrictEqual(coerceToType(null, "date"), null); - assert.deepStrictEqual(coerceToType("", "date"), null); - assert.deepStrictEqual(coerceToType(" ", "date"), null); - assert.deepStrictEqual( - coerceToType({toString: () => " "}, "date").toString(), - invalidDate.toString() - ); - assert.deepStrictEqual( - coerceToType({toString: () => "2020-01-01"}, "date"), - new Date("2020-01-01") - ); - }); - - it.skip("coerces to string", () => { - assert.deepStrictEqual(coerceToType(true, "string"), "true"); - assert.deepStrictEqual(coerceToType(false, "string"), "false"); - assert.deepStrictEqual(coerceToType(10, "string"), "10"); - assert.deepStrictEqual(coerceToType({a: 1}, "string"), "[object Object]"); - assert.deepStrictEqual(coerceToType(0, "string"), "0"); - assert.deepStrictEqual(coerceToType("", "string"), ""); - assert.deepStrictEqual(coerceToType(" ", "string"), " "); - assert.deepStrictEqual(coerceToType(" foo", "string"), " foo"); - assert.deepStrictEqual(coerceToType(" foo ", "string"), " foo "); - assert.deepStrictEqual(coerceToType(null, "string"), null); - assert.deepStrictEqual(coerceToType(undefined, "string"), undefined); - assert.deepStrictEqual(coerceToType(NaN, "string"), "NaN"); - }); - - it.skip("coerces to bigint", () => { - assert.deepStrictEqual(coerceToType("32", "bigint"), 32n); - assert.deepStrictEqual(coerceToType(" 32", "bigint"), 32n); - assert.deepStrictEqual(coerceToType(32n, "bigint"), 32n); - assert.deepStrictEqual(coerceToType(0, "bigint"), 0n); - assert.deepStrictEqual(coerceToType(false, "bigint"), 0n); - assert.deepStrictEqual(coerceToType(true, "bigint"), 1n); - assert.deepStrictEqual(coerceToType(null, "bigint"), null); - assert.deepStrictEqual(coerceToType(undefined, "bigint"), undefined); - assert.deepStrictEqual(coerceToType(1.1, "bigint"), undefined); - assert.deepStrictEqual(coerceToType("1.1", "bigint"), undefined); - assert.deepStrictEqual(coerceToType(" 32n", "bigint"), undefined); - assert.deepStrictEqual(coerceToType("A", "bigint"), undefined); - assert.deepStrictEqual(coerceToType("", "bigint"), undefined); - assert.deepStrictEqual(coerceToType(" ", "bigint"), undefined); - assert.deepStrictEqual(coerceToType(NaN, "bigint"), undefined); + assert.deepStrictEqual(coerceValue("12/12/2020", "date"), new Date("12/12/2020")); + assert.deepStrictEqual(coerceValue("12/12/2020 ", "date"), new Date("12/12/2020")); + assert.deepStrictEqual(coerceValue("2022-01-01T12:34:00Z", "date"), new Date("2022-01-01T12:34:00Z")); // prettier-ignore + assert.deepStrictEqual(coerceValue("{a: 1}", "date"), invalidDate); + assert.deepStrictEqual(coerceValue("true", "date"), invalidDate); + assert.deepStrictEqual(coerceValue("2020-1-12", "date"), invalidDate); + assert.deepStrictEqual(coerceValue("1675356739000", "date"), invalidDate); + assert.deepStrictEqual(coerceValue("undefined", "date"), invalidDate); + assert.deepStrictEqual(coerceValue("", "date"), null); + }); + it("coerces to string", () => { + assert.deepStrictEqual(coerceValue("true", "string"), "true"); + assert.deepStrictEqual(coerceValue("false", "string"), "false"); + assert.deepStrictEqual(coerceValue("10", "string"), "10"); + assert.deepStrictEqual(coerceValue("", "string"), ""); + assert.deepStrictEqual(coerceValue(" ", "string"), " "); }); - - it.skip("coerces to array", () => { - // "array" is not a target type for coercion, but can be inferred. - assert.deepStrictEqual(coerceToType([1, 2, 3], "array"), [1, 2, 3]); - assert.deepStrictEqual(coerceToType(null, "array"), null); - assert.deepStrictEqual(coerceToType(undefined, "array"), undefined); - }); - - it.skip("coerces to object", () => { - // "object" is not a target type for coercion, but can be inferred. - assert.deepStrictEqual(coerceToType({a: 1, b: 2}, "object"), {a: 1, b: 2}); - assert.deepStrictEqual(coerceToType(null, "object"), null); - assert.deepStrictEqual(coerceToType(undefined, "object"), undefined); - }); - - it.skip("coerces to buffer", () => { - // "buffer" is not a target type for coercion, but can be inferred. - assert.deepStrictEqual(coerceToType(new ArrayBuffer(), "buffer"), new ArrayBuffer()); - assert.deepStrictEqual(coerceToType("A", "buffer"), "A"); - assert.deepStrictEqual(coerceToType(null, "buffer"), null); - assert.deepStrictEqual(coerceToType(undefined, "buffer"), undefined); - }); - - it.skip("coerces to other", () => { - // "other" is not a target type for coercion, but can be inferred. - assert.deepStrictEqual(coerceToType(0, "other"), 0); - assert.deepStrictEqual(coerceToType("a", "other"), "a"); - assert.deepStrictEqual(coerceToType(null, "other"), null); - assert.deepStrictEqual(coerceToType(undefined, "other"), undefined); - }); - - // Note: if type is "raw", coerceToType() will not be called. Instead, values - // will be returned from coerceRow(). }); diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts index ef024c9..fe345a6 100644 --- a/src/runtime/stdlib/dsv.ts +++ b/src/runtime/stdlib/dsv.ts @@ -1,179 +1,98 @@ -import type {ColumnSchema} from "./databaseClient.js"; +/* eslint-disable @typescript-eslint/no-unused-expressions */ + +export type Type = "boolean" | "integer" | "number" | "date" | "string"; + +export interface ColumnSchema { + /** The name of the column. */ + name: string; + /** The type of the column. */ + type: Type; +} /** Accepts dates in the form of ISOString and LocaleDateString, with or without time. */ const DATE_TEST = /^(([-+]\d{2})?\d{4}(-\d{2}(-\d{2}))|(\d{1,2})\/(\d{1,2})\/(\d{2,4}))([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore +/** The maximum number of rows to sample (including missing values). */ +const SAMPLE_SIZE = 100; + export function inferSchema( - source: Record[], - columns = getAllKeys(source) + rows: Record[], + columns = getAllKeys(rows) ): ColumnSchema[] { const schema: ColumnSchema[] = []; - const sampleSize = 100; - const sample = source.slice(0, sampleSize); - const typeCounts: Record = {}; - for (const col of columns) { - const colCount = (typeCounts[col] = createTypeCount()); - for (const d of sample) { - let value = d[col]; - if (value == null) continue; - // const type = typeof value; - // if (type !== "string") { - // ++colCount.defined; - // if (Array.isArray(value)) ++colCount.array; - // else if (value instanceof Date) ++colCount.date; - // else if (value instanceof ArrayBuffer) ++colCount.buffer; - // else if (type === "number") { - // ++colCount.number; - // if (Number.isInteger(value)) ++colCount.integer; - // } - // // bigint, boolean, or object - // else if (type in colCount) ++colCount[type]; - // } else { - value = value.trim(); + const n = Math.min(rows.length, SAMPLE_SIZE); + for (const column of columns) { + let booleans = 0; + let integers = 0; + let numbers = 0; + let dates = 0; + let strings = 0; + + for (let i = 0; i < n; ++i) { + const value = rows[i][column]?.trim(); if (!value) continue; - ++colCount.defined; - ++colCount.string; + ++strings; if (/^(true|false)$/i.test(value)) { - ++colCount.boolean; - } else if (value && !isNaN(Number(value))) { - ++colCount.number; - if (Number.isInteger(+value)) ++colCount.integer; - } else if (DATE_TEST.test(value)) ++colCount.date; - // } - } - // Chose the non-string, non-other type with the greatest count that is also - // ≥90%; or if no such type meets that criterion, fall back to string if - // ≥90%; and lastly fall back to other. - const minCount = Math.max(1, Math.ceil(colCount.defined * 0.9)); - let type: Type = "string"; - let typeCount = minCount - 1; - console.warn(colCount, typeCount) - for (const inferrableType of inferrableTypes) { - if (colCount[inferrableType] > typeCount) { - type = inferrableType; - typeCount = colCount[type]; + ++booleans; + } else if (!isNaN(Number(value))) { + ++numbers; + if (Number.isInteger(+value)) ++integers; + } else if (DATE_TEST.test(value)) { + ++dates; } } - // if (colCount.string > typeCount) { - // type = "string"; - // typeCount = colCount.string; - // } - schema.push({name: col, type}); + + // Chose the non-string type with the greatest count that is also ≥90%; or + // if no such type meets that criterion, use string. Note: integer is more + // specific than number, and hence should be tested before number. + let type: Type = "string"; + let typeCount = Math.max(1, Math.ceil(strings * 0.9)) - 1; + if (booleans > typeCount) ((type = "boolean"), (typeCount = booleans)); + if (integers > typeCount) ((type = "integer"), (typeCount = integers)); + if (numbers > typeCount) ((type = "number"), (typeCount = numbers)); + if (dates > typeCount) ((type = "date"), (typeCount = dates)); + schema.push({name: column, type}); } return schema; } export function enforceSchema( - source: Record[], + rows: Record[], schema: ColumnSchema[] ): Record[] & {schema: typeof schema} { - const types = new Map(schema.map(({name, type}) => [name, type])); - return Object.assign( - source.map((d) => coerceRow(d, types, schema)), - {schema} - ); + return Object.assign(rows.map((row) => coerceRow(row, schema)), {schema}); // prettier-ignore } export function coerceRow( - object: Record, - types: Map, + input: Record, schema: ColumnSchema[] -) { - const coerced: Record = {}; - for (const col of schema) { - const type = types.get(col.name)!; - const value = object[col.name]; - coerced[col.name] = type === "raw" ? value : coerceToType(value, type); - } - return coerced; +): Record { + const output: Record = {}; + for (const {name, type} of schema) output[name] = coerceValue(input[name], type); + return output; } -export function coerceToType(value: unknown, type: Type): unknown { +export function coerceValue(value: string, type: Type): unknown { switch (type) { - case "string": - return typeof value === "string" || value == null ? value : String(value); - case "boolean": - if (typeof value === "string") { - const trimValue = value.trim().toLowerCase(); - return trimValue === "true" ? true : trimValue === "false" ? false : null; - } - return typeof value === "boolean" || value == null ? value : Boolean(value); - case "bigint": - return typeof value === "bigint" || value == null - ? value - : Number.isInteger(typeof value === "string" && !value.trim() ? NaN : +value) - ? BigInt(value as string) - : undefined; - case "integer": // not a target type for coercion, but can be inferred + case "boolean": { + const trimmed = value.trim().toLowerCase(); + return trimmed === "true" ? true : trimmed === "false" ? false : null; + } + case "integer": case "number": { - return typeof value === "number" - ? value - : value == null || (typeof value === "string" && !value.trim()) - ? NaN - : Number(value); + const trimmed = value.trim(); + return trimmed ? Number(value) : NaN; } case "date": { - if (value instanceof Date || value == null) return value; - if (typeof value === "number") return new Date(value); - const trimValue = String(value).trim(); - if (typeof value === "string" && !trimValue) return null; - return new Date(DATE_TEST.test(trimValue) ? trimValue : NaN); + const trimmed = value.trim(); + return trimmed ? new Date(DATE_TEST.test(trimmed) ? trimmed : NaN) : null; } - case "array": - case "object": - case "buffer": - case "other": + default: { return value; - default: - throw new Error(`Unable to coerce to type: ${type}`); + } } } -function createTypeCount(): TypeCount { - return { - boolean: 0, - integer: 0, - number: 0, - date: 0, - string: 0, - array: 0, - object: 0, - bigint: 0, - buffer: 0, - defined: 0 - }; -} - -type TypeCount = Record | "defined", number>; - -export type Type = - | "boolean" - | "integer" - | "number" - | "date" - | "bigint" - | "array" - | "object" - | "buffer" - | "string" - | "other"; - -// Caution: the order below matters! 🌶️ The first one that passes the ≥90% test -// should be the one that we chose, and therefore these types should be listed -// from most specific to least specific. -const inferrableTypes: Exclude[] = [ - "boolean", - "integer", - "number", - "date", - // "string" - // "bigint", - // "array", - // "object", - // "buffer" - // Note: "other" and "string" are intentionally omitted; see below! -]; - export function getAllKeys(rows: Record[]): string[] { const keys = new Set(); for (const row of rows) { From 20fbc638c4841721edc3ad4cfd3c90d4285bdd3e Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 15 Jun 2026 21:32:44 -0600 Subject: [PATCH 4/9] even simpler --- src/runtime/stdlib/dsv.test.ts | 183 ++++++++++++--------------------- src/runtime/stdlib/dsv.ts | 108 ++++++------------- 2 files changed, 96 insertions(+), 195 deletions(-) diff --git a/src/runtime/stdlib/dsv.test.ts b/src/runtime/stdlib/dsv.test.ts index 89368de..107f4a3 100644 --- a/src/runtime/stdlib/dsv.test.ts +++ b/src/runtime/stdlib/dsv.test.ts @@ -1,141 +1,88 @@ import {assert, describe, it} from "vitest"; -import type {Type} from "./dsv"; -import {coerceValue, enforceSchema, inferSchema} from "./dsv"; +import {inferTypes} from "./dsv"; +import {coerceBoolean, coerceDate, coerceNumber} from "./dsv"; -function inferType(values: string[]): Type { - return inferSchema(values.map((value) => ({value})))[0].type; +function inferType(values: string[]): unknown[] { + return inferTypes(values.map((value) => ({value})), ["value"]).map(({value}) => value); // prettier-ignore } -function enforceType(values: string[], type: Type): unknown[] { - return enforceSchema(values.map((value) => ({value})), [{name: "value", type}]).map(({value}) => value); // prettier-ignore -} - -describe("inferSchema", () => { - it("infers the union of columns", () => { - assert.deepStrictEqual(inferSchema([{a: "2"}, {b: "bee"}, {c: "true"}]), [ - {name: "a", type: "integer"}, - {name: "b", type: "string"}, - {name: "c", type: "boolean"} - ]); - }); - it("infers integers", () => { - assert.deepStrictEqual(inferType(["2", "4", "6"]), "integer"); - }); +describe("inferTypes", () => { it("infers numbers", () => { - assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), "number"); - }); - it("infers numbers with no leading zero", () => { - assert.deepStrictEqual(inferType([".2", ".4", ".67"]), "number"); + assert.deepStrictEqual(inferType(["2", "4", "6"]), [2, 4, 6]); + assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), [1.2, 3.4, 5.67]); + assert.deepStrictEqual(inferType([".2", ".4", ".67"]), [0.2, 0.4, 0.67]); + assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9].map(String)), [0.1, 1, 2, 3, 9]); + assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), [NaN, 1, 2, 3, 9, 1, 1, 1, 3, 2, 3]); // prettier-ignore + assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), [0, 0.1, 2, 3, 4, 5, 2, 3, 0.9, NaN]); // prettier-ignore }); it("infers booleans", () => { - assert.deepStrictEqual(inferType(["true", "", "false"]), "boolean"); + assert.deepStrictEqual(inferType(["true", "", "false"]), [true, null, false]); + assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, false, "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, false, null]); // prettier-ignore }); it("infers dates in common formats", () => { - assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), "date"); + assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T07:00:00.000Z"), new Date("2020-11-12T19:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore }); it("infers strings", () => { - assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), "string"); - }); - it('considers "42n" to be a string, not a bigint', () => { - assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), "string"); - }); - it("considers (exclusively) empty strings to be strings", () => { - assert.deepStrictEqual(inferType(["", ""]), "string"); - }); - it("considers mixed integers and numbers to be numbers, not integers", () => { - assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9, 10, 11, 12, 13].map(String)), "number"); - }); - it('considers "NaN" to be a string, not a number', () => { - assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), "string"); - }); - it("infers mostly integers as integers", () => { - assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), "integer"); - }); - it("infers partly integers as strings", () => { - assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), "string"); - }); - it("infers mostly numbers as numbers", () => { - assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), "number"); // prettier-ignore - }); - it("infers partly numbers as strings", () => { - assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), "string"); - }); - it("infers mostly booleans as booleans", () => { - assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, false, "pants on fire"].map(String)), "boolean"); // prettier-ignore - }); - it("infers partly booleans as strings", () => { - assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), "string"); // prettier-ignore + assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), ["cat", "dog", "1,000", "null"]); // prettier-ignore + assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), ["10n", "22n", "0n"]); + assert.deepStrictEqual(inferType(["", ""]), ["", ""]); + assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), ["NaN", "NaN", "NaN", "1", "2", "3"]); // prettier-ignore + assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), ["x", 1, 2, 3, 9, 1, 1, 3].map(String)); // prettier-ignore + assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), [".0", ".1", "2", "3", "4", "5", "2", "3", "x"]); // prettier-ignore + assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), ["true", "false", "true", "false", "true", "false", "pants on fire"]); // prettier-ignore }); }); -describe("enforceSchema", () => { - it("enforces integers", () => { - assert.deepStrictEqual(enforceType(["1", "2", "4", "3", ""], "integer"), [1, 2, 4, 3, NaN]); - }); - it("enforces numbers", () => { - assert.deepStrictEqual(enforceType(["1.2", "4.3", "0.1", ""], "number"), [1.2, 4.3, 0.1, NaN]); - }); - it("enforces booleans", () => { - assert.deepStrictEqual(enforceType(["true", "false", ""], "boolean"), [true, false, null]); - }); - it("enforces dates", () => { - assert.deepStrictEqual(enforceType(["2002-01-01", "2003-01-01", ""], "date"), [new Date("2002-01-01"), new Date("2003-01-01"), null]); // prettier-ignore +describe("coerceNumber", () => { + it("coerces to number", () => { + assert.deepStrictEqual(coerceNumber("1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2); + assert.deepStrictEqual(coerceNumber("10"), 10); + assert.deepStrictEqual(coerceNumber("0"), 0); + assert.deepStrictEqual(coerceNumber("A"), NaN); + assert.deepStrictEqual(coerceNumber(""), NaN); + assert.deepStrictEqual(coerceNumber(" "), NaN); + assert.deepStrictEqual(coerceNumber("null"), NaN); + assert.deepStrictEqual(coerceNumber("1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2); + assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2); + assert.deepStrictEqual(coerceNumber("0"), 0); + assert.deepStrictEqual(coerceNumber(""), NaN); + assert.deepStrictEqual(coerceNumber(" "), NaN); + assert.deepStrictEqual(coerceNumber("A"), NaN); + assert.deepStrictEqual(coerceNumber("null"), NaN); + assert.deepStrictEqual(coerceNumber("undefined"), NaN); + assert.deepStrictEqual(coerceNumber("{a: 1}"), NaN); }); }); -describe("coerceValue", () => { - it("coerces to integer", () => { - assert.deepStrictEqual(coerceValue("1.2", "integer"), 1.2); - assert.deepStrictEqual(coerceValue(" 1.2", "integer"), 1.2); - assert.deepStrictEqual(coerceValue(" 1.2 ", "integer"), 1.2); - assert.deepStrictEqual(coerceValue("10", "integer"), 10); - assert.deepStrictEqual(coerceValue("0", "integer"), 0); - assert.deepStrictEqual(coerceValue("A", "integer"), NaN); - assert.deepStrictEqual(coerceValue("", "integer"), NaN); - assert.deepStrictEqual(coerceValue(" ", "integer"), NaN); - assert.deepStrictEqual(coerceValue("null", "integer"), NaN); - }); - it("coerces to number", () => { - assert.deepStrictEqual(coerceValue("1.2", "number"), 1.2); - assert.deepStrictEqual(coerceValue(" 1.2", "number"), 1.2); - assert.deepStrictEqual(coerceValue(" 1.2 ", "number"), 1.2); - assert.deepStrictEqual(coerceValue("0", "number"), 0); - assert.deepStrictEqual(coerceValue("", "number"), NaN); - assert.deepStrictEqual(coerceValue(" ", "number"), NaN); - assert.deepStrictEqual(coerceValue("A", "number"), NaN); - assert.deepStrictEqual(coerceValue("null", "number"), NaN); - assert.deepStrictEqual(coerceValue("undefined", "number"), NaN); - assert.deepStrictEqual(coerceValue("{a: 1}", "number"), NaN); - }); +describe("coerceBoolean", () => { it("coerces to boolean", () => { - assert.deepStrictEqual(coerceValue("true", "boolean"), true); - assert.deepStrictEqual(coerceValue("True ", "boolean"), true); - assert.deepStrictEqual(coerceValue("False", "boolean"), false); - assert.deepStrictEqual(coerceValue("false", "boolean"), false); - assert.deepStrictEqual(coerceValue("1", "boolean"), null); // TODO null vs. undefined? - assert.deepStrictEqual(coerceValue("2", "boolean"), null); - assert.deepStrictEqual(coerceValue("0", "boolean"), null); - assert.deepStrictEqual(coerceValue("{}", "boolean"), null); - assert.deepStrictEqual(coerceValue("null", "boolean"), null); - assert.deepStrictEqual(coerceValue("undefined", "boolean"), null); + assert.deepStrictEqual(coerceBoolean("true"), true); + assert.deepStrictEqual(coerceBoolean("True "), true); + assert.deepStrictEqual(coerceBoolean("False"), false); + assert.deepStrictEqual(coerceBoolean("false"), false); + assert.deepStrictEqual(coerceBoolean("1"), null); // TODO null vs. undefined? + assert.deepStrictEqual(coerceBoolean("2"), null); + assert.deepStrictEqual(coerceBoolean("0"), null); + assert.deepStrictEqual(coerceBoolean("{}"), null); + assert.deepStrictEqual(coerceBoolean("null"), null); + assert.deepStrictEqual(coerceBoolean("undefined"), null); }); +}); + +describe("coerceDate", () => { it("coerces to date", () => { const invalidDate = new Date(NaN); - assert.deepStrictEqual(coerceValue("12/12/2020", "date"), new Date("12/12/2020")); - assert.deepStrictEqual(coerceValue("12/12/2020 ", "date"), new Date("12/12/2020")); - assert.deepStrictEqual(coerceValue("2022-01-01T12:34:00Z", "date"), new Date("2022-01-01T12:34:00Z")); // prettier-ignore - assert.deepStrictEqual(coerceValue("{a: 1}", "date"), invalidDate); - assert.deepStrictEqual(coerceValue("true", "date"), invalidDate); - assert.deepStrictEqual(coerceValue("2020-1-12", "date"), invalidDate); - assert.deepStrictEqual(coerceValue("1675356739000", "date"), invalidDate); - assert.deepStrictEqual(coerceValue("undefined", "date"), invalidDate); - assert.deepStrictEqual(coerceValue("", "date"), null); - }); - it("coerces to string", () => { - assert.deepStrictEqual(coerceValue("true", "string"), "true"); - assert.deepStrictEqual(coerceValue("false", "string"), "false"); - assert.deepStrictEqual(coerceValue("10", "string"), "10"); - assert.deepStrictEqual(coerceValue("", "string"), ""); - assert.deepStrictEqual(coerceValue(" ", "string"), " "); + assert.deepStrictEqual(coerceDate("12/12/2020"), new Date("12/12/2020")); + assert.deepStrictEqual(coerceDate("12/12/2020 "), new Date("12/12/2020")); + assert.deepStrictEqual(coerceDate("2022-01-01T12:34:00Z"), new Date("2022-01-01T12:34:00Z")); // prettier-ignore + assert.deepStrictEqual(coerceDate("{a: 1}"), invalidDate); + assert.deepStrictEqual(coerceDate("true"), invalidDate); + assert.deepStrictEqual(coerceDate("2020-1-12"), invalidDate); + assert.deepStrictEqual(coerceDate("1675356739000"), invalidDate); + assert.deepStrictEqual(coerceDate("undefined"), invalidDate); + assert.deepStrictEqual(coerceDate(""), null); }); }); diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts index fe345a6..0d28b2e 100644 --- a/src/runtime/stdlib/dsv.ts +++ b/src/runtime/stdlib/dsv.ts @@ -1,106 +1,60 @@ /* eslint-disable @typescript-eslint/no-unused-expressions */ -export type Type = "boolean" | "integer" | "number" | "date" | "string"; - -export interface ColumnSchema { - /** The name of the column. */ - name: string; - /** The type of the column. */ - type: Type; -} - /** Accepts dates in the form of ISOString and LocaleDateString, with or without time. */ const DATE_TEST = /^(([-+]\d{2})?\d{4}(-\d{2}(-\d{2}))|(\d{1,2})\/(\d{1,2})\/(\d{2,4}))([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore /** The maximum number of rows to sample (including missing values). */ const SAMPLE_SIZE = 100; -export function inferSchema( - rows: Record[], - columns = getAllKeys(rows) -): ColumnSchema[] { - const schema: ColumnSchema[] = []; - const n = Math.min(rows.length, SAMPLE_SIZE); +export function inferTypes>( + rows: T[], + columns: (keyof T)[] +): Record[] { + const output = rows as Record[]; + const n = rows.length; + const k = Math.min(n, SAMPLE_SIZE); for (const column of columns) { let booleans = 0; - let integers = 0; let numbers = 0; let dates = 0; let strings = 0; - for (let i = 0; i < n; ++i) { + for (let i = 0; i < k; ++i) { const value = rows[i][column]?.trim(); if (!value) continue; ++strings; - if (/^(true|false)$/i.test(value)) { - ++booleans; - } else if (!isNaN(Number(value))) { - ++numbers; - if (Number.isInteger(+value)) ++integers; - } else if (DATE_TEST.test(value)) { - ++dates; - } + if (/^(true|false)$/i.test(value)) ++booleans; + else if (!isNaN(Number(value))) ++numbers; + else if (DATE_TEST.test(value)) ++dates; } // Chose the non-string type with the greatest count that is also ≥90%; or - // if no such type meets that criterion, use string. Note: integer is more - // specific than number, and hence should be tested before number. - let type: Type = "string"; + // if no such type meets that criterion, use string. + let coerce: ((value: string) => unknown) | undefined = undefined; let typeCount = Math.max(1, Math.ceil(strings * 0.9)) - 1; - if (booleans > typeCount) ((type = "boolean"), (typeCount = booleans)); - if (integers > typeCount) ((type = "integer"), (typeCount = integers)); - if (numbers > typeCount) ((type = "number"), (typeCount = numbers)); - if (dates > typeCount) ((type = "date"), (typeCount = dates)); - schema.push({name: column, type}); - } - return schema; -} + if (booleans > typeCount) ((coerce = coerceBoolean), (typeCount = booleans)); + if (numbers > typeCount) ((coerce = coerceNumber), (typeCount = numbers)); + if (dates > typeCount) ((coerce = coerceDate), (typeCount = dates)); + if (!coerce) continue; -export function enforceSchema( - rows: Record[], - schema: ColumnSchema[] -): Record[] & {schema: typeof schema} { - return Object.assign(rows.map((row) => coerceRow(row, schema)), {schema}); // prettier-ignore + for (let i = 0; i < n; ++i) { + output[i][column] = coerce(rows[i][column]); + } + } + return output; } -export function coerceRow( - input: Record, - schema: ColumnSchema[] -): Record { - const output: Record = {}; - for (const {name, type} of schema) output[name] = coerceValue(input[name], type); - return output; +export function coerceBoolean(value: string): boolean | null { + const trimmed = value.trim().toLowerCase(); + return trimmed === "true" ? true : trimmed === "false" ? false : null; } -export function coerceValue(value: string, type: Type): unknown { - switch (type) { - case "boolean": { - const trimmed = value.trim().toLowerCase(); - return trimmed === "true" ? true : trimmed === "false" ? false : null; - } - case "integer": - case "number": { - const trimmed = value.trim(); - return trimmed ? Number(value) : NaN; - } - case "date": { - const trimmed = value.trim(); - return trimmed ? new Date(DATE_TEST.test(trimmed) ? trimmed : NaN) : null; - } - default: { - return value; - } - } +export function coerceNumber(value: string): number { + const trimmed = value.trim(); + return trimmed ? Number(value) : NaN; } -export function getAllKeys(rows: Record[]): string[] { - const keys = new Set(); - for (const row of rows) { - if (row != null) { - for (const key of Object.keys(row)) { - keys.add(key); - } - } - } - return Array.from(keys); +export function coerceDate(value: string): Date | null { + const trimmed = value.trim(); + return trimmed ? new Date(DATE_TEST.test(trimmed) ? trimmed : NaN) : null; } From 4b2d8543a61ef15a080297e4cceba49598467db1 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 15 Jun 2026 21:35:55 -0600 Subject: [PATCH 5/9] distinguish invalid vs. empty boolean --- src/runtime/stdlib/dsv.test.ts | 14 +++++++------- src/runtime/stdlib/dsv.ts | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/runtime/stdlib/dsv.test.ts b/src/runtime/stdlib/dsv.test.ts index 107f4a3..6af95d5 100644 --- a/src/runtime/stdlib/dsv.test.ts +++ b/src/runtime/stdlib/dsv.test.ts @@ -17,7 +17,7 @@ describe("inferTypes", () => { }); it("infers booleans", () => { assert.deepStrictEqual(inferType(["true", "", "false"]), [true, null, false]); - assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, false, "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, false, null]); // prettier-ignore + assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, "", "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, null, undefined]); // prettier-ignore }); it("infers dates in common formats", () => { assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T07:00:00.000Z"), new Date("2020-11-12T19:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore @@ -63,12 +63,12 @@ describe("coerceBoolean", () => { assert.deepStrictEqual(coerceBoolean("True "), true); assert.deepStrictEqual(coerceBoolean("False"), false); assert.deepStrictEqual(coerceBoolean("false"), false); - assert.deepStrictEqual(coerceBoolean("1"), null); // TODO null vs. undefined? - assert.deepStrictEqual(coerceBoolean("2"), null); - assert.deepStrictEqual(coerceBoolean("0"), null); - assert.deepStrictEqual(coerceBoolean("{}"), null); - assert.deepStrictEqual(coerceBoolean("null"), null); - assert.deepStrictEqual(coerceBoolean("undefined"), null); + assert.deepStrictEqual(coerceBoolean(""), null); + assert.deepStrictEqual(coerceBoolean("1"), undefined); + assert.deepStrictEqual(coerceBoolean("2"), undefined); + assert.deepStrictEqual(coerceBoolean("0"), undefined); + assert.deepStrictEqual(coerceBoolean("null"), undefined); + assert.deepStrictEqual(coerceBoolean("undefined"), undefined); }); }); diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts index 0d28b2e..747d163 100644 --- a/src/runtime/stdlib/dsv.ts +++ b/src/runtime/stdlib/dsv.ts @@ -44,9 +44,9 @@ export function inferTypes>( return output; } -export function coerceBoolean(value: string): boolean | null { +export function coerceBoolean(value: string): boolean | null | undefined { const trimmed = value.trim().toLowerCase(); - return trimmed === "true" ? true : trimmed === "false" ? false : null; + return trimmed === "true" ? true : trimmed === "false" ? false : trimmed ? undefined : null; } export function coerceNumber(value: string): number { From b83f1d91e366b5c5af6fae1b00ac75c4ce65cd69 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 15 Jun 2026 21:38:44 -0600 Subject: [PATCH 6/9] typed: "auto" --- src/runtime/stdlib/fileAttachment.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/runtime/stdlib/fileAttachment.ts b/src/runtime/stdlib/fileAttachment.ts index b448646..cbc868b 100644 --- a/src/runtime/stdlib/fileAttachment.ts +++ b/src/runtime/stdlib/fileAttachment.ts @@ -1,7 +1,9 @@ +import {inferTypes} from "./dsv.js"; + /* eslint-disable @typescript-eslint/no-explicit-any */ const files = new Map(); -export type DsvOptions = {delimiter?: string; array?: boolean; typed?: boolean}; +export type DsvOptions = {delimiter?: string; array?: boolean; typed?: "auto" | boolean}; export type DsvResult = (Record[] | any[][]) & {columns: string[]}; export interface FileAttachment { @@ -136,11 +138,13 @@ export abstract class AbstractFile implements FileAttachment { async stream(): Promise>> { return (await fetchFile(this)).body!; } - async dsv({delimiter = ",", array = false, typed = false} = {}): Promise { + async dsv({delimiter = ",", array = false, typed = false}: DsvOptions = {}): Promise { const [text, d3] = await Promise.all([this.text(), import("https://cdn.jsdelivr.net/npm/d3-dsv/+esm")]); // prettier-ignore const format = d3.dsvFormat(delimiter); - const parse = array ? format.parseRows : format.parse; - return parse(text, typed && d3.autoType); + const parse = array ? ((typed &&= true), format.parseRows) : format.parse; + const output = parse(text, typed === true ? d3.autoType : undefined); + if (typed === "auto") inferTypes(output, output.columns); + return output; } async csv(options?: Omit): Promise { return this.dsv({...options, delimiter: ","}); From 52fbd8bd4af733124244b6e0ccdee2759710fbca Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 15 Jun 2026 21:40:56 -0600 Subject: [PATCH 7/9] fix time zones --- package.json | 2 +- src/runtime/stdlib/dsv.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 98180a3..4dfb8f5 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "version": "2.1.7", "type": "module", "scripts": { - "test": "vitest", + "test": "TZ=America/Los_Angeles vitest", "prepublishOnly": "rm -rf dist && tsc && chmod +x dist/bin/*.js && cp -r src/styles src/templates dist/src && cp src/runtime/stdlib/*.css dist/src/runtime/stdlib", "lint": "tsc --noEmit && eslint bin src types", "notebooks": "tsx bin/notebooks.ts", diff --git a/src/runtime/stdlib/dsv.test.ts b/src/runtime/stdlib/dsv.test.ts index 6af95d5..b54fcb0 100644 --- a/src/runtime/stdlib/dsv.test.ts +++ b/src/runtime/stdlib/dsv.test.ts @@ -20,7 +20,7 @@ describe("inferTypes", () => { assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, "", "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, null, undefined]); // prettier-ignore }); it("infers dates in common formats", () => { - assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T07:00:00.000Z"), new Date("2020-11-12T19:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore + assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T08:00:00.000Z"), new Date("2020-11-12T20:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore }); it("infers strings", () => { assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), ["cat", "dog", "1,000", "null"]); // prettier-ignore From fcbf1434b2a07373446dd1e8c4a55323a6f7a025 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 16 Jun 2026 17:01:41 -0600 Subject: [PATCH 8/9] remove pointless capturing groups --- src/runtime/stdlib/dsv.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts index 747d163..fb1d3f0 100644 --- a/src/runtime/stdlib/dsv.ts +++ b/src/runtime/stdlib/dsv.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions */ /** Accepts dates in the form of ISOString and LocaleDateString, with or without time. */ -const DATE_TEST = /^(([-+]\d{2})?\d{4}(-\d{2}(-\d{2}))|(\d{1,2})\/(\d{1,2})\/(\d{2,4}))([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore +const DATE_TEST = /^([-+]\d{2})?\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{2,4}([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore /** The maximum number of rows to sample (including missing values). */ const SAMPLE_SIZE = 100; From 0651d4714fe281e19e355374ad60eaa25c122bdf Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 16 Jun 2026 17:15:09 -0600 Subject: [PATCH 9/9] document DATE_TEST better --- src/runtime/stdlib/dsv.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/runtime/stdlib/dsv.ts b/src/runtime/stdlib/dsv.ts index fb1d3f0..889cec1 100644 --- a/src/runtime/stdlib/dsv.ts +++ b/src/runtime/stdlib/dsv.ts @@ -1,6 +1,21 @@ /* eslint-disable @typescript-eslint/no-unused-expressions */ -/** Accepts dates in the form of ISOString and LocaleDateString, with or without time. */ +/** + * Accepts dates in the following forms: + * - ±YYYYYY-MM-DD + * - YYYY-MM-DD + * - MM/DD/YY + * - MM/DD/YYYY + * + * Following a "T" or a space, dates may additionally have a time: + * - HH:MM + * - HH:MM:SS + * - HH:MM:SS.MMM + * + * And the time may optionally have a time zone: + * - Z + * - ±HH:MM + */ const DATE_TEST = /^([-+]\d{2})?\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{2,4}([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore /** The maximum number of rows to sample (including missing values). */