Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

annotate local paths #1731

Merged
merged 18 commits into from
Oct 28, 2024
31 changes: 18 additions & 13 deletions src/javascript/transpile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ export async function transpileModule(

async function rewriteImportSource(source: StringLiteral) {
const specifier = getStringLiteralValue(source);
output.replaceLeft(source.start, source.end, JSON.stringify(await resolveImport(specifier)));
output.replaceLeft(source.start, source.end, annotatePath(await resolveImport(specifier)));
}

for (const {name, node} of findFiles(body, path, input)) {
Expand All @@ -111,17 +111,15 @@ export async function transpileModule(
output.replaceLeft(
source.start,
source.end,
`${JSON.stringify(
`${
info
? {
name: p,
mimeType: mime.getType(name) ?? undefined,
path: relativePath(servePath, resolveFile(name)),
lastModified: info.mtimeMs,
size: info.size
}
: p
)}, import.meta.url`
? `{"name":${JSON.stringify(p)},"mimeType":${JSON.stringify(
mime.getType(name) ?? undefined
)},"path":${annotatePath(relativePath(servePath, resolveFile(name)))},"lastModified":${JSON.stringify(
info.mtimeMs
)},"size":${JSON.stringify(info.size)}}`
: JSON.stringify(p)
}, import.meta.url`
);
}

Expand All @@ -137,7 +135,7 @@ export async function transpileModule(
if (isImportMetaResolve(node) && isStringLiteral(source)) {
const value = getStringLiteralValue(source);
const resolution = isPathImport(value) && !isJavaScript(value) ? resolveFile(value) : await resolveImport(value);
output.replaceLeft(source.start, source.end, JSON.stringify(resolution));
output.replaceLeft(source.start, source.end, annotatePath(resolution));
}
}

Expand Down Expand Up @@ -205,7 +203,7 @@ function rewriteImportDeclarations(
for (const node of declarations) {
output.delete(node.start, node.end + +(output.input[node.end] === "\n"));
specifiers.push(rewriteImportSpecifiers(node));
imports.push(`import(${JSON.stringify(resolve(getStringLiteralValue(node.source as StringLiteral)))})`);
imports.push(`import(${annotatePath(resolve(getStringLiteralValue(node.source as StringLiteral)))})`);
}
if (declarations.length > 1) {
output.insertLeft(0, `const [${specifiers.join(", ")}] = await Promise.all([${imports.join(", ")}]);\n`);
Expand Down Expand Up @@ -259,3 +257,10 @@ export function rewriteParams(output: Sourcemap, body: Node, params: Params, inp
output.replaceLeft(node.start, node.end, JSON.stringify(params[name]));
}
}

/**
* Annotate a path to a local import or file so it can be reworked server-side.
*/
export function annotatePath(uri: string): string {
return `${JSON.stringify(uri)}${isPathImport(uri) ? "/* observablehq-file */" : ""}`;
}
3 changes: 2 additions & 1 deletion src/node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import esbuild from "rollup-plugin-esbuild";
import {prepareOutput, toOsPath} from "./files.js";
import type {ImportReference} from "./javascript/imports.js";
import {isJavaScript, parseImports} from "./javascript/imports.js";
import {annotatePath} from "./javascript/transpile.js";
import {parseNpmSpecifier, rewriteNpmImports} from "./npm.js";
import {isPathImport, relativePath} from "./path.js";
import {faint} from "./tty.js";
Expand Down Expand Up @@ -86,7 +87,7 @@ function isBadCommonJs(specifier: string): boolean {
}

function shimCommonJs(specifier: string, require: NodeRequire): string {
return `export {${Object.keys(require(specifier))}} from ${JSON.stringify(specifier)};\n`;
return `export {${Object.keys(require(specifier))}} from ${annotatePath(specifier)};\n`;
}

async function bundle(
Expand Down
3 changes: 2 additions & 1 deletion src/npm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {isImportMetaResolve, parseImports} from "./javascript/imports.js";
import {parseProgram} from "./javascript/parse.js";
import type {StringLiteral} from "./javascript/source.js";
import {getStringLiteralValue, isStringLiteral} from "./javascript/source.js";
import {annotatePath} from "./javascript/transpile.js";
import {relativePath} from "./path.js";
import {Sourcemap} from "./sourcemap.js";
import {faint, yellow} from "./tty.js";
Expand Down Expand Up @@ -64,7 +65,7 @@ export function rewriteNpmImports(input: string, resolve: (s: string) => string
const value = getStringLiteralValue(source);
const resolved = resolve(value);
if (resolved === undefined || value === resolved) return;
output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
output.replaceLeft(source.start, source.end, annotatePath(resolved));
}

// TODO Preserve the source map, but download it too.
Expand Down
3 changes: 2 additions & 1 deletion src/rollup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import esbuild from "rollup-plugin-esbuild";
import {getClientPath, getStylePath} from "./files.js";
import type {StringLiteral} from "./javascript/source.js";
import {getStringLiteralValue, isStringLiteral} from "./javascript/source.js";
import {annotatePath} from "./javascript/transpile.js";
import {resolveNpmImport} from "./npm.js";
import {getObservableUiOrigin} from "./observableApiClient.js";
import {isAssetPath, isPathImport, relativePath} from "./path.js";
Expand Down Expand Up @@ -177,7 +178,7 @@ function importMetaResolve(path: string, resolveImport: ImportResolver): Plugin
for (const source of resolves) {
const specifier = getStringLiteralValue(source);
const resolution = await resolveImport(specifier);
if (resolution) output.replaceLeft(source.start, source.end, JSON.stringify(relativePath(path, resolution)));
if (resolution) output.replaceLeft(source.start, source.end, annotatePath(relativePath(path, resolution)));
}

return {code: String(output)};
Expand Down
4 changes: 2 additions & 2 deletions test/javascript/transpile-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,11 +210,11 @@ describe("transpileModule(input, root, path)", () => {
it("rewrites npm imports", async () => {
const input = 'import "npm:d3-array";';
const output = (await transpileModule(input, options)).split("\n").pop()!;
assert.strictEqual(output, 'import "../_npm/[email protected]/_esm.js";');
assert.strictEqual(output, 'import "../_npm/[email protected]/_esm.js"/* observablehq-file */;');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this link is tagged as observable-file, then we should also rewrite it to tag Observable files in it. If we don't rewrite it, I don't think we should tag it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was initially doing that (see line 307 in 6cfb74d#diff-a479e6903ca67c840829983998e33849155911842cf08fa118b42b2d88fe3c9b), then I think we decided to tag everything and let the server decide what it should do with it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My concern is that this is a reference to an _npm file that is tagged, but references to _npm files from other _npm files are not tagged. As a concrete example, in the docs build there is this chain:

  1. dist/chart.js tags its import to the _import file:
  2. docs/.observablehq/dist/_import/chart.95e875cf.js tags its import of an _npm file:
  3. docs/.observablehq/dist/_npm/@observablehq/[email protected]/e828d8c8.js does not tag its import of the _npm file:
  4. ../../[email protected]/c68fbd73.js

It would be helpful if the tagging was done based on the taget of the URL, not the place the URL occurs. As it stands, the server can't really make the decision fully, because links to _npm files aren't consistent tagged/not tagged, so it has to special case them as not needing extra authentication.

});
it("rewrites node imports", async () => {
const input = 'import "d3-array";';
const output = (await transpileModule(input, options)).split("\n").pop()!;
assert.strictEqual(output, 'import "../_node/[email protected]/index.js";');
assert.strictEqual(output, 'import "../_node/[email protected]/index.js"/* observablehq-file */;');
});
});
30 changes: 15 additions & 15 deletions test/npm-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,37 +103,37 @@ describe("fromJsDelivrPath(path)", () => {
// prettier-ignore
describe("rewriteNpmImports(input, resolve)", () => {
it("rewrites /npm/ imports to /_npm/", () => {
assert.strictEqual(rewriteNpmImports('export * from "/npm/[email protected]/dist/d3-array.js";\n', (v) => resolve("/_npm/[email protected]/dist/d3.js", v)), 'export * from "../../[email protected]/dist/d3-array.js";\n');
assert.strictEqual(rewriteNpmImports('export * from "/npm/[email protected]/dist/d3-array.js";\n', (v) => resolve("/_npm/[email protected]/dist/d3.js", v)), 'export * from "../../[email protected]/dist/d3-array.js"/* observablehq-file */;\n');
});
it("rewrites /npm/…+esm imports to _esm.js", () => {
assert.strictEqual(rewriteNpmImports('export * from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'export * from "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('export * from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'export * from "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites /npm/ imports to a relative path", () => {
assert.strictEqual(rewriteNpmImports('import "/npm/[email protected]/dist/d3-array.js";\n', (v) => resolve("/_npm/[email protected]/dist/d3.js", v)), 'import "../../[email protected]/dist/d3-array.js";\n');
assert.strictEqual(rewriteNpmImports('import "/npm/[email protected]/dist/d3-array.js";\n', (v) => resolve("/_npm/[email protected]/d3.js", v)), 'import "../[email protected]/dist/d3-array.js";\n');
assert.strictEqual(rewriteNpmImports('import "/npm/[email protected]/dist/d3-array.js";\n', (v) => resolve("/_npm/[email protected]/dist/d3.js", v)), 'import "../../[email protected]/dist/d3-array.js"/* observablehq-file */;\n');
assert.strictEqual(rewriteNpmImports('import "/npm/[email protected]/dist/d3-array.js";\n', (v) => resolve("/_npm/[email protected]/d3.js", v)), 'import "../[email protected]/dist/d3-array.js"/* observablehq-file */;\n');
});
it("rewrites named imports", () => {
assert.strictEqual(rewriteNpmImports('import {sort} from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import {sort} from "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('import {sort} from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import {sort} from "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites empty imports", () => {
assert.strictEqual(rewriteNpmImports('import "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('import "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites default imports", () => {
assert.strictEqual(rewriteNpmImports('import d3 from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import d3 from "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('import d3 from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import d3 from "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites namespace imports", () => {
assert.strictEqual(rewriteNpmImports('import * as d3 from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import * as d3 from "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('import * as d3 from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import * as d3 from "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites named exports", () => {
assert.strictEqual(rewriteNpmImports('export {sort} from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'export {sort} from "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('export {sort} from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'export {sort} from "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites namespace exports", () => {
assert.strictEqual(rewriteNpmImports('export * from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'export * from "../[email protected]/_esm.js";\n');
assert.strictEqual(rewriteNpmImports('export * from "/npm/[email protected]/+esm";\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'export * from "../[email protected]/_esm.js"/* observablehq-file */;\n');
});
it("rewrites dynamic imports with static module specifiers", () => {
assert.strictEqual(rewriteNpmImports('import("/npm/[email protected]/+esm");\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js");\n');
assert.strictEqual(rewriteNpmImports("import(`/npm/[email protected]/+esm`);\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js");\n');
assert.strictEqual(rewriteNpmImports("import('/npm/[email protected]/+esm');\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js");\n');
assert.strictEqual(rewriteNpmImports('import("/npm/[email protected]/+esm");\n', (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js"/* observablehq-file */);\n');
assert.strictEqual(rewriteNpmImports("import(`/npm/[email protected]/+esm`);\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js"/* observablehq-file */);\n');
assert.strictEqual(rewriteNpmImports("import('/npm/[email protected]/+esm');\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js"/* observablehq-file */);\n');
});
it("ignores dynamic imports with dynamic module specifiers", () => {
assert.strictEqual(rewriteNpmImports("import(`/npm/d3-array@${version}/+esm`);\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), "import(`/npm/d3-array@${version}/+esm`);\n");
Expand All @@ -142,8 +142,8 @@ describe("rewriteNpmImports(input, resolve)", () => {
assert.strictEqual(rewriteNpmImports("import(`/npm/d3-array@${version}/+esm`);\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), "import(`/npm/d3-array@${version}/+esm`);\n");
});
it("strips the sourceMappingURL declaration", () => {
assert.strictEqual(rewriteNpmImports("import(`/npm/[email protected]/+esm`);\n//# sourceMappingURL=index.js.map", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js");\n');
assert.strictEqual(rewriteNpmImports("import(`/npm/[email protected]/+esm`);\n//# sourceMappingURL=index.js.map\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js");\n');
assert.strictEqual(rewriteNpmImports("import(`/npm/[email protected]/+esm`);\n//# sourceMappingURL=index.js.map", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js"/* observablehq-file */);\n');
assert.strictEqual(rewriteNpmImports("import(`/npm/[email protected]/+esm`);\n//# sourceMappingURL=index.js.map\n", (v) => resolve("/_npm/[email protected]/_esm.js", v)), 'import("../[email protected]/_esm.js"/* observablehq-file */);\n');
});
});

Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
export {test} from "./test.86a60bc6.js";
export {test} from "./test.86a60bc6.js"/* observablehq-file */;
4 changes: 2 additions & 2 deletions test/output/build/data-loaders/_import/test.86a60bc6.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
import {FileAttachment} from "../_observablehq/stdlib.00000003.js";
import {FileAttachment} from "../_observablehq/stdlib.00000003.js"/* observablehq-file */;

export const test = FileAttachment({"name":"../test.txt","mimeType":"text/plain","path":"../_file/test.f2ca1bb6.txt","lastModified":/* ts */1706742000000,"size":5}, import.meta.url).text();
export const test = FileAttachment({"name":"../test.txt","mimeType":"text/plain","path":"../_file/test.f2ca1bb6.txt"/* observablehq-file */,"lastModified":/* ts */1706742000000,"size":5}, import.meta.url).text();
2 changes: 1 addition & 1 deletion test/output/build/data-loaders/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
registerFile("./test.txt", {"name":"./test.txt","mimeType":"text/plain","path":"./_file/test.f2ca1bb6.txt","lastModified":/* ts */1706742000000,"size":5});

define({id: "05e74070", inputs: ["display"], outputs: ["test"], body: async (display) => {
const {test} = await import("./_import/import-test.e7269c4e.js");
const {test} = await import("./_import/import-test.e7269c4e.js"/* observablehq-file */);

display(await test);
return {test};
Expand Down
6 changes: 3 additions & 3 deletions test/output/build/embed/_import/chart.2ce91e05.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import {FileAttachment} from "../_observablehq/stdlib.00000003.js";
import * as Plot from "../_npm/@observablehq/[email protected]/cd372fb8.js";
import {FileAttachment} from "../_observablehq/stdlib.00000003.js"/* observablehq-file */;
import * as Plot from "../_npm/@observablehq/[email protected]/cd372fb8.js"/* observablehq-file */;

export async function Chart() {
const gistemp = await FileAttachment({"name":"../lib/gistemp.csv","mimeType":"text/csv","path":"../_file/lib/gistemp.1cf298b1.csv","lastModified":/* ts */1706742000000,"size":97}, import.meta.url).csv({typed: true});
const gistemp = await FileAttachment({"name":"../lib/gistemp.csv","mimeType":"text/csv","path":"../_file/lib/gistemp.1cf298b1.csv"/* observablehq-file */,"lastModified":/* ts */1706742000000,"size":97}, import.meta.url).csv({typed: true});
return Plot.plot({
y: {grid: true},
color: {scheme: "burd"},
Expand Down
8 changes: 4 additions & 4 deletions test/output/build/embed/chart.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import "./_observablehq/stdlib.00000003.js";
import "./_npm/@observablehq/[email protected]/cd372fb8.js";
import "./_npm/[email protected]/cd372fb8.js";
export * from "./_import/chart.2ce91e05.js";
import "./_observablehq/stdlib.00000003.js"/* observablehq-file */;
import "./_npm/@observablehq/[email protected]/cd372fb8.js"/* observablehq-file */;
import "./_npm/[email protected]/cd372fb8.js"/* observablehq-file */;
export * from "./_import/chart.2ce91e05.js"/* observablehq-file */;
6 changes: 3 additions & 3 deletions test/output/build/fetches/_import/foo/foo.666599bc.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions test/output/build/fetches/_import/top.c85e149a.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion test/output/build/fetches/foo.html
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
registerFile("./foo/foo-data.json", {"name":"./foo/foo-data.json","mimeType":"application/json","path":"./_file/foo/foo-data.67358ed8.json","lastModified":/* ts */1706742000000,"size":10});

define({id: "47a695da", inputs: ["display"], outputs: ["fooJsonData","fooCsvData"], body: async (display) => {
const {fooJsonData, fooCsvData} = await import("./_import/foo/foo.666599bc.js");
const {fooJsonData, fooCsvData} = await import("./_import/foo/foo.666599bc.js"/* observablehq-file */);

display(fooJsonData);
display(fooCsvData);
Expand Down
2 changes: 1 addition & 1 deletion test/output/build/fetches/top.html
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
registerFile("./top-data.json", {"name":"./top-data.json","mimeType":"application/json","path":"./_file/top-data.67358ed8.json","lastModified":/* ts */1706742000000,"size":10});

define({id: "cb908c08", inputs: ["display"], outputs: ["fooCsvData","fooJsonData","topCsvData","topJsonData"], body: async (display) => {
const {fooCsvData, fooJsonData, topCsvData, topJsonData} = await import("./_import/top.c85e149a.js");
const {fooCsvData, fooJsonData, topCsvData, topJsonData} = await import("./_import/top.c85e149a.js"/* observablehq-file */);

display(fooJsonData);
display(fooCsvData);
Expand Down
2 changes: 1 addition & 1 deletion test/output/build/imports/_import/bar/bar.4460ccc2.js
Original file line number Diff line number Diff line change
@@ -1 +1 @@
export {bar} from "./baz.2add1dd0.js";
export {bar} from "./baz.2add1dd0.js"/* observablehq-file */;
2 changes: 1 addition & 1 deletion test/output/build/imports/_import/bar/baz.2add1dd0.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {foo} from "../foo/foo.bcd720b2.js";
import {foo} from "../foo/foo.bcd720b2.js"/* observablehq-file */;

export const bar = "bar";
export const foobar = foo + "bar";
Loading