FileAttachment.csv({typed: "auto"})

package.json

@@ -8,7 +8,7 @@
   "version": "2.1.7",
   "type": "module",
   "scripts": {
-    "test": "vitest",
+    "test": "TZ=America/Los_Angeles vitest",
     "prepublishOnly": "rm -rf dist && tsc && chmod +x dist/bin/*.js && cp -r src/styles src/templates dist/src && cp src/runtime/stdlib/*.css dist/src/runtime/stdlib",
     "lint": "tsc --noEmit && eslint bin src types",
     "notebooks": "tsx bin/notebooks.ts",

src/runtime/stdlib/dsv.test.ts

@@ -0,0 +1,88 @@
+import {assert, describe, it} from "vitest";
+import {inferTypes} from "./dsv";
+import {coerceBoolean, coerceDate, coerceNumber} from "./dsv";
+
+function inferType(values: string[]): unknown[] {
+  return inferTypes(values.map((value) => ({value})), ["value"]).map(({value}) => value); // prettier-ignore
+}
+
+describe("inferTypes", () => {
+  it("infers numbers", () => {
+    assert.deepStrictEqual(inferType(["2", "4", "6"]), [2, 4, 6]);
+    assert.deepStrictEqual(inferType(["1.2", "3.4", "5.67"]), [1.2, 3.4, 5.67]);
+    assert.deepStrictEqual(inferType([".2", ".4", ".67"]), [0.2, 0.4, 0.67]);
+    assert.deepStrictEqual(inferType([0.1, 1, 2, 3, 9].map(String)), [0.1, 1, 2, 3, 9]);
+    assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 1, 3, 2, 3].map(String)), [NaN, 1, 2, 3, 9, 1, 1, 1, 3, 2, 3]); // prettier-ignore
+    assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", ".9", "x"]), [0, 0.1, 2, 3, 4, 5, 2, 3, 0.9, NaN]); // prettier-ignore
+  });
+  it("infers booleans", () => {
+    assert.deepStrictEqual(inferType(["true", "", "false"]), [true, null, false]);
+    assert.deepStrictEqual(inferType([true, false, true, false, true, false, true, false, true, "", "pants on fire"].map(String)), [true, false, true, false, true, false, true, false, true, null, undefined]); // prettier-ignore
+  });
+  it("infers dates in common formats", () => {
+    assert.deepStrictEqual(inferType(["1/2/20", "2020-11-12 12:23:00", "", "2020-01-12"]), [new Date("2020-01-02T08:00:00.000Z"), new Date("2020-11-12T20:23:00.000Z"), null, new Date("2020-01-12T00:00:00.000Z")]); // prettier-ignore
+  });
+  it("infers strings", () => {
+    assert.deepStrictEqual(inferType(["cat", "dog", "1,000", "null"]), ["cat", "dog", "1,000", "null"]); // prettier-ignore
+    assert.deepStrictEqual(inferType(["10n", "22n", "0n"]), ["10n", "22n", "0n"]);
+    assert.deepStrictEqual(inferType(["", ""]), ["", ""]);
+    assert.deepStrictEqual(inferType(["NaN", "NaN", "NaN", "1", "2", "3"]), ["NaN", "NaN", "NaN", "1", "2", "3"]); // prettier-ignore
+    assert.deepStrictEqual(inferType(["x", 1, 2, 3, 9, 1, 1, 3].map(String)), ["x", 1, 2, 3, 9, 1, 1, 3].map(String)); // prettier-ignore
+    assert.deepStrictEqual(inferType([".0", ".1", "2", "3", "4", "5", "2", "3", "x"]), [".0", ".1", "2", "3", "4", "5", "2", "3", "x"]); // prettier-ignore
+    assert.deepStrictEqual(inferType([true, false, true, false, true, false, "pants on fire"].map(String)), ["true", "false", "true", "false", "true", "false", "pants on fire"]); // prettier-ignore
+  });
+});
+
+describe("coerceNumber", () => {
+  it("coerces to number", () => {
+    assert.deepStrictEqual(coerceNumber("1.2"), 1.2);
+    assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2);
+    assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2);
+    assert.deepStrictEqual(coerceNumber("10"), 10);
+    assert.deepStrictEqual(coerceNumber("0"), 0);
+    assert.deepStrictEqual(coerceNumber("A"), NaN);
+    assert.deepStrictEqual(coerceNumber(""), NaN);
+    assert.deepStrictEqual(coerceNumber(" "), NaN);
+    assert.deepStrictEqual(coerceNumber("null"), NaN);
+    assert.deepStrictEqual(coerceNumber("1.2"), 1.2);
+    assert.deepStrictEqual(coerceNumber(" 1.2"), 1.2);
+    assert.deepStrictEqual(coerceNumber(" 1.2 "), 1.2);
+    assert.deepStrictEqual(coerceNumber("0"), 0);
+    assert.deepStrictEqual(coerceNumber(""), NaN);
+    assert.deepStrictEqual(coerceNumber(" "), NaN);
+    assert.deepStrictEqual(coerceNumber("A"), NaN);
+    assert.deepStrictEqual(coerceNumber("null"), NaN);
+    assert.deepStrictEqual(coerceNumber("undefined"), NaN);
+    assert.deepStrictEqual(coerceNumber("{a: 1}"), NaN);
+  });
+});
+
+describe("coerceBoolean", () => {
+  it("coerces to boolean", () => {
+    assert.deepStrictEqual(coerceBoolean("true"), true);
+    assert.deepStrictEqual(coerceBoolean("True   "), true);
+    assert.deepStrictEqual(coerceBoolean("False"), false);
+    assert.deepStrictEqual(coerceBoolean("false"), false);
+    assert.deepStrictEqual(coerceBoolean(""), null);
+    assert.deepStrictEqual(coerceBoolean("1"), undefined);
+    assert.deepStrictEqual(coerceBoolean("2"), undefined);
+    assert.deepStrictEqual(coerceBoolean("0"), undefined);
+    assert.deepStrictEqual(coerceBoolean("null"), undefined);
+    assert.deepStrictEqual(coerceBoolean("undefined"), undefined);
+  });
+});
+
+describe("coerceDate", () => {
+  it("coerces to date", () => {
+    const invalidDate = new Date(NaN);
+    assert.deepStrictEqual(coerceDate("12/12/2020"), new Date("12/12/2020"));
+    assert.deepStrictEqual(coerceDate("12/12/2020  "), new Date("12/12/2020"));
+    assert.deepStrictEqual(coerceDate("2022-01-01T12:34:00Z"), new Date("2022-01-01T12:34:00Z")); // prettier-ignore
+    assert.deepStrictEqual(coerceDate("{a: 1}"), invalidDate);
+    assert.deepStrictEqual(coerceDate("true"), invalidDate);
+    assert.deepStrictEqual(coerceDate("2020-1-12"), invalidDate);
+    assert.deepStrictEqual(coerceDate("1675356739000"), invalidDate);
+    assert.deepStrictEqual(coerceDate("undefined"), invalidDate);
+    assert.deepStrictEqual(coerceDate(""), null);
+  });
+});

src/runtime/stdlib/dsv.ts

@@ -0,0 +1,75 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+
+/**
+ * Accepts dates in the following forms:
+ * - ±YYYYYY-MM-DD
+ * - YYYY-MM-DD
+ * - MM/DD/YY
+ * - MM/DD/YYYY
+ *
+ * Following a "T" or a space, dates may additionally have a time:
+ * - HH:MM
+ * - HH:MM:SS
+ * - HH:MM:SS.MMM
+ *
+ * And the time may optionally have a time zone:
+ * - Z
+ * - ±HH:MM
+ */
+const DATE_TEST = /^([-+]\d{2})?\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{2,4}([T ]\d{2}:\d{2}(:\d{2}(\.\d{3})?)?(Z|[-+]\d{2}:\d{2})?)?$/; // prettier-ignore
+
+/** The maximum number of rows to sample (including missing values). */
+const SAMPLE_SIZE = 100;
+
+export function inferTypes<T extends Record<string, string>>(
+  rows: T[],
+  columns: (keyof T)[]
+): Record<keyof T, unknown>[] {
+  const output = rows as Record<keyof T, unknown>[];
+  const n = rows.length;
+  const k = Math.min(n, SAMPLE_SIZE);
+  for (const column of columns) {
+    let booleans = 0;
+    let numbers = 0;
+    let dates = 0;
+    let strings = 0;
+
+    for (let i = 0; i < k; ++i) {
+      const value = rows[i][column]?.trim();
+      if (!value) continue;
+      ++strings;
+      if (/^(true|false)$/i.test(value)) ++booleans;
+      else if (!isNaN(Number(value))) ++numbers;
+      else if (DATE_TEST.test(value)) ++dates;
+    }
+
+    // Chose the non-string type with the greatest count that is also ≥90%; or
+    // if no such type meets that criterion, use string.
+    let coerce: ((value: string) => unknown) | undefined = undefined;
+    let typeCount = Math.max(1, Math.ceil(strings * 0.9)) - 1;
+    if (booleans > typeCount) ((coerce = coerceBoolean), (typeCount = booleans));
+    if (numbers > typeCount) ((coerce = coerceNumber), (typeCount = numbers));
+    if (dates > typeCount) ((coerce = coerceDate), (typeCount = dates));
+    if (!coerce) continue;
+
+    for (let i = 0; i < n; ++i) {
+      output[i][column] = coerce(rows[i][column]);
+    }
+  }
+  return output;
+}
+
+export function coerceBoolean(value: string): boolean | null | undefined {
+  const trimmed = value.trim().toLowerCase();
+  return trimmed === "true" ? true : trimmed === "false" ? false : trimmed ? undefined : null;
+}
+
+export function coerceNumber(value: string): number {
+  const trimmed = value.trim();
+  return trimmed ? Number(value) : NaN;
+}
+
+export function coerceDate(value: string): Date | null {
+  const trimmed = value.trim();
+  return trimmed ? new Date(DATE_TEST.test(trimmed) ? trimmed : NaN) : null;
+}

src/runtime/stdlib/fileAttachment.ts

@@ -1,7 +1,9 @@
+import {inferTypes} from "./dsv.js";
+
 /* eslint-disable @typescript-eslint/no-explicit-any */
 const files = new Map<string, FileAttachmentImpl>();
 
-export type DsvOptions = {delimiter?: string; array?: boolean; typed?: boolean};
+export type DsvOptions = {delimiter?: string; array?: boolean; typed?: "auto" | boolean};
 export type DsvResult = (Record<string, any>[] | any[][]) & {columns: string[]};
 
 export interface FileAttachment {
@@ -136,11 +138,13 @@ export abstract class AbstractFile implements FileAttachment {
   async stream(): Promise<ReadableStream<Uint8Array<ArrayBufferLike>>> {
     return (await fetchFile(this)).body!;
   }
-  async dsv({delimiter = ",", array = false, typed = false} = {}): Promise<DsvResult> {
+  async dsv({delimiter = ",", array = false, typed = false}: DsvOptions = {}): Promise<DsvResult> {
     const [text, d3] = await Promise.all([this.text(), import("https://cdn.jsdelivr.net/npm/d3-dsv/+esm")]); // prettier-ignore
     const format = d3.dsvFormat(delimiter);
-    const parse = array ? format.parseRows : format.parse;
-    return parse(text, typed && d3.autoType);
+    const parse = array ? ((typed &&= true), format.parseRows) : format.parse;
+    const output = parse(text, typed === true ? d3.autoType : undefined);
+    if (typed === "auto") inferTypes(output, output.columns);
+    return output;
   }
   async csv(options?: Omit<DsvOptions, "delimiter">): Promise<DsvResult> {
     return this.dsv({...options, delimiter: ","});