Add file.csv({typed: "auto"}) option that uses inferSchema (#360)

* add new typed: auto option that uses inferSchema and coerceRow

* matches d3 api slightly better i guess

* update readme

* Don't do typed: auto if array option is passed

Co-authored-by: Mike Bostock <[email protected]>

* check for columns in inferSchema; pull out enforceSchema function which propagates schema

* test that inferSchema looks at source.columns

* test enforceSchema

* dont pass columns now that its the internal default of inferSchema

* fewer defaults, more explicit

---------

Co-authored-by: Mike Bostock <[email protected]>

Author: tophtucker

README.md

@@ -332,7 +332,7 @@ Returns a promise to the file’s contents, parsed as tab-separated values (TSV)
 const data = await FileAttachment("cars.tsv").tsv();
 ```
 
-If <i>array</i> is true, an array of arrays is returned; otherwise, the first row is assumed to be the header row and an array of objects is returned, and the returned array has a <i>data</i>.columns property that is an array of column names. (See <a href="https://github.com/d3/d3-dsv/blob/main/README.md#dsv_parseRows">d3.tsvParseRows</a>.) If <i>typed</i> is true, [automatic type inference](https://observablehq.com/@d3/d3-autotype) is applied; only use this feature if you know your data is compatible.
+If <i>array</i> is true, an array of arrays is returned; otherwise, the first row is assumed to be the header row and an array of objects is returned, and the returned array has a <i>data</i>.columns property that is an array of column names. (See <a href="https://github.com/d3/d3-dsv/blob/main/README.md#dsv_parseRows">d3.tsvParseRows</a>.) If <i>typed</i> is true, [automatic type inference](https://observablehq.com/@d3/d3-autotype) is applied to each row independently; if <i>typed</i> is “auto”, the type inference is based on a sample of rows. Only use this feature if you know your data is compatible.
 
 <a href="#attachment_image" name="attachment_image">#</a> *attachment*.<b>image</b>(<i>options</i>) [<>](https://github.com/observablehq/stdlib/blob/main/src/fileAttachment.js "Source")
 

src/fileAttachment.js

@@ -2,6 +2,7 @@ import {autoType, csvParse, csvParseRows, tsvParse, tsvParseRows} from "d3-dsv";
 import {arrow4, arrow9, arrow11, jszip, exceljs} from "./dependencies.js";
 import {cdn, requireDefault} from "./require.js";
 import {SQLiteDatabaseClient} from "./sqlite.js";
+import {inferSchema, coerceRow} from "./table.js";
 import {Workbook} from "./xlsx.js";
 
 async function remote_fetch(file) {
@@ -10,11 +11,21 @@ async function remote_fetch(file) {
   return response;
 }
 
+export function enforceSchema(source, schema) {
+  const types = new Map(schema.map(({name, type}) => [name, type]));
+  return Object.assign(source.map(d => coerceRow(d, types, schema)), {schema});
+}
+
 async function dsv(file, delimiter, {array = false, typed = false} = {}) {
   const text = await file.text();
-  return (delimiter === "\t"
-      ? (array ? tsvParseRows : tsvParse)
-      : (array ? csvParseRows : csvParse))(text, typed && autoType);
+  const parse = (delimiter === "\t"
+    ? (array ? tsvParseRows : tsvParse)
+    : (array ? csvParseRows : csvParse));
+  if (typed === "auto" && !array) {
+    const source = parse(text);
+    return enforceSchema(source, inferSchema(source, source.columns));
+  }
+  return parse(text, typed && autoType);
 }
 
 export class AbstractFile {

src/table.js

@@ -199,8 +199,8 @@ function sourceCache(loadSource) {
 const loadChartDataSource = sourceCache(async (source) => {
   if (source instanceof FileAttachment) {
     switch (source.mimeType) {
-      case "text/csv": return source.csv({typed: true});
-      case "text/tab-separated-values": return source.tsv({typed: true});
+      case "text/csv": return source.csv({typed: "auto"});
+      case "text/tab-separated-values": return source.tsv({typed: "auto"});
       case "application/json": return source.json();
     }
     throw new Error(`unsupported file type: ${source.mimeType}`);
@@ -626,7 +626,7 @@ export function __table(source, operations) {
   let {schema, columns} = source;
   let inferredSchema = false;
   if (!isQueryResultSetSchema(schema)) {
-    schema = inferSchema(source, columns);
+    schema = inferSchema(source, isQueryResultSetColumns(columns) ? columns : undefined);
     inferredSchema = true;
   }
   // Combine column types from schema with user-selected types in operations
@@ -785,7 +785,7 @@ export function __table(source, operations) {
   return source;
 }
 
-function coerceRow(object, types, schema) {
+export function coerceRow(object, types, schema) {
   const coerced = {};
   for (const col of schema) {
     const type = types.get(col.name);

test/fileAttachments-test.js

@@ -1,5 +1,7 @@
 import assert from "assert";
+import {enforceSchema} from "../src/fileAttachment.js";
 import {FileAttachments} from "../src/index.js";
+import {inferSchema} from "../src/table.js";
 
 it("FileAttachments is exported by stdlib", () => {
   assert.strictEqual(typeof FileAttachments, "function");
@@ -38,3 +40,30 @@ it("FileAttachment works with Promises that resolve to URLs", async () => {
   assert.strictEqual(file.constructor.name, "FileAttachment");
   assert.strictEqual(await file.url(), "https://example.com/otherfile.js");
 });
+
+it("enforceSchema coerces an array of objects", () => {
+  const source = [{a: "0", b: "1", c: "2"}];
+  assert.deepStrictEqual(
+    enforceSchema(source, inferSchema(source)),
+    Object.assign(
+      [{a: 0, b: 1, c: 2}],
+      {schema: [
+        {
+          inferred: "integer",
+          name: "a",
+          type: "integer"
+        },
+        {
+          inferred: "integer",
+          name: "b",
+          type: "integer"
+        },
+        {
+          inferred: "integer",
+          name: "c",
+          type: "integer"
+        }
+      ]}
+    )
+  );
+});