diff --git a/packages/host/app/components/operator-mode/edit-field-modal.gts b/packages/host/app/components/operator-mode/edit-field-modal.gts index d95d0535d9..35b1b15a4d 100644 --- a/packages/host/app/components/operator-mode/edit-field-modal.gts +++ b/packages/host/app/components/operator-mode/edit-field-modal.gts @@ -28,7 +28,7 @@ import { catalogEntryRef, CodeRef, } from '@cardstack/runtime-common'; -import { makeResolvedURL } from '@cardstack/runtime-common/loader'; + import type { ModuleSyntax } from '@cardstack/runtime-common/module-syntax'; import ModalContainer from '@cardstack/host/components/modal-container'; @@ -236,8 +236,8 @@ export default class EditFieldModal extends Component { fieldType, fieldDefinitionType: this.isFieldDef ? 'field' : 'card', incomingRelativeTo, - outgoingRelativeTo: this.loaderService.loader.reverseResolution( - makeResolvedURL(this.operatorModeStateService.state.codePath!).href, + outgoingRelativeTo: new URL( + this.operatorModeStateService.state.codePath!, ), outgoingRealmURL: new URL(this.args.file.realmURL), addFieldAtIndex, diff --git a/packages/host/app/services/loader-service.ts b/packages/host/app/services/loader-service.ts index edb76b407d..6beb96727e 100644 --- a/packages/host/app/services/loader-service.ts +++ b/packages/host/app/services/loader-service.ts @@ -43,7 +43,7 @@ export default class LoaderService extends Service { } let loader = this.virtualNetwork.createLoader(); - loader.addURLMapping( + this.virtualNetwork.addURLMapping( new URL(baseRealm.url), new URL(config.resolvedBaseRealmURL), ); diff --git a/packages/host/app/services/message-service.ts b/packages/host/app/services/message-service.ts index ba99097484..e23585c8ae 100644 --- a/packages/host/app/services/message-service.ts +++ b/packages/host/app/services/message-service.ts @@ -1,4 +1,5 @@ import Service, { service } from '@ember/service'; + import { tracked } from '@glimmer/tracking'; import type LoaderService from './loader-service'; @@ -12,13 +13,13 @@ export default class MessageService extends Service { } subscribe(realmURL: string, cb: (ev: MessageEvent) => void): () => void { - let resolvedRealmURL = this.loaderService.loader.resolve(realmURL); - let maybeEventSource = this.subscriptions.get(resolvedRealmURL.href); + let maybeEventSource = this.subscriptions.get(realmURL); if (!maybeEventSource) { - maybeEventSource = new EventSource(resolvedRealmURL); + maybeEventSource = + this.loaderService.virtualNetwork.createEventSource(realmURL); maybeEventSource.onerror = () => eventSource.close(); - this.subscriptions.set(resolvedRealmURL.href, maybeEventSource); + this.subscriptions.set(realmURL, maybeEventSource); } let eventSource = maybeEventSource; diff --git a/packages/host/app/services/operator-mode-state-service.ts b/packages/host/app/services/operator-mode-state-service.ts index 3729006030..e7f8417a82 100644 --- a/packages/host/app/services/operator-mode-state-service.ts +++ b/packages/host/app/services/operator-mode-state-service.ts @@ -257,8 +257,8 @@ export default class OperatorModeStateService extends Service { } get codePathRelativeToRealm() { - if (this.state.codePath && this.resolvedRealmURL) { - let realmPath = new RealmPaths(this.resolvedRealmURL); + if (this.state.codePath && this.realmURL) { + let realmPath = new RealmPaths(this.realmURL); if (realmPath.inRealm(this.state.codePath)) { try { @@ -488,10 +488,6 @@ export default class OperatorModeStateService extends Service { return this.cardService.defaultURL; } - get resolvedRealmURL() { - return this.loaderService.loader.resolve(this.realmURL); - } - subscribeToOpenFileStateChanges(subscriber: OpenFileSubscriber) { this.openFileSubscribers.push(subscriber); } diff --git a/packages/host/app/services/recent-files-service.ts b/packages/host/app/services/recent-files-service.ts index b1b6e4cfd5..76f9e7655b 100644 --- a/packages/host/app/services/recent-files-service.ts +++ b/packages/host/app/services/recent-files-service.ts @@ -52,7 +52,7 @@ export default class RecentFilesService extends Service { } // TODO this wont work when visiting files that come from multiple realms in // code mode... - let realmURL = this.operatorModeStateService.resolvedRealmURL; + let realmURL = this.operatorModeStateService.realmURL; if (realmURL) { let realmPaths = new RealmPaths(new URL(realmURL)); diff --git a/packages/host/tests/acceptance/code-submode/recent-files-test.ts b/packages/host/tests/acceptance/code-submode/recent-files-test.ts index 7b5275bf7b..29e617b46e 100644 --- a/packages/host/tests/acceptance/code-submode/recent-files-test.ts +++ b/packages/host/tests/acceptance/code-submode/recent-files-test.ts @@ -498,7 +498,7 @@ module('Acceptance | code submode | recent files tests', function (hooks) { ], ], submode: 'code', - codePath: `http://localhost:4201/base/code-ref.gts`, + codePath: `https://cardstack.com/base/date.gts`, fileView: 'browser', openDirs: {}, }); @@ -527,9 +527,8 @@ module('Acceptance | code submode | recent files tests', function (hooks) { await waitFor('[data-test-file="field-component.gts"]'); await click('[data-test-file="field-component.gts"]'); await waitFor('[data-test-file="field-component.gts"].selected'); - assert - .dom('[data-test-recent-file]:nth-child(1)') - .containsText('field-component.gts'); + + assert.dom('[data-test-recent-file]:nth-child(1)').containsText('date.gts'); assert .dom('[data-test-recent-file]:nth-child(2)') .containsText('code-ref.gts'); @@ -541,6 +540,7 @@ module('Acceptance | code submode | recent files tests', function (hooks) { JSON.parse(window.localStorage.getItem('recent-files') || '[]'), [ ['https://cardstack.com/base/', 'field-component.gts'], + ['https://cardstack.com/base/', 'date.gts'], ['https://cardstack.com/base/', 'code-ref.gts'], ['https://cardstack.com/base/', 'catalog-entry.gts'], ], diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index 297edc8d38..0e5504b0f2 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -39,6 +39,7 @@ import CardPrerender from '@cardstack/host/components/card-prerender'; import type CardService from '@cardstack/host/services/card-service'; import type { CardSaveSubscriber } from '@cardstack/host/services/card-service'; +import type LoaderService from '@cardstack/host/services/loader-service'; import type MessageService from '@cardstack/host/services/message-service'; import { @@ -448,7 +449,6 @@ export async function setupIntegrationTestRealm({ export const testRealmSecretSeed = "shhh! it's a secret"; async function setupTestRealm({ - loader, contents, realmURL, onFetch, @@ -466,6 +466,9 @@ async function setupTestRealm({ permissions?: RealmPermissions; }) { let owner = (getContext() as TestContext).owner; + let { loader, virtualNetwork } = owner.lookup( + 'service:loader-service', + ) as LoaderService; realmURL = realmURL ?? testRealmURL; @@ -548,6 +551,7 @@ async function setupTestRealm({ realmSecretSeed: testRealmSecretSeed, }); + virtualNetwork.mount(realm.maybeHandle); await realm.ready; return { realm, adapter }; } diff --git a/packages/host/tests/integration/components/text-input-validator-test.gts b/packages/host/tests/integration/components/text-input-validator-test.gts index 8404183e2d..ec2772ae6d 100644 --- a/packages/host/tests/integration/components/text-input-validator-test.gts +++ b/packages/host/tests/integration/components/text-input-validator-test.gts @@ -67,10 +67,6 @@ module('Integration | text-input-validator', function (hooks) { loader = (this.owner.lookup('service:loader-service') as LoaderService) .loader; - loader.addURLMapping( - new URL(baseRealm.url), - new URL('http://localhost:4201/base/'), - ); cardApi = await loader.import(`${baseRealm.url}card-api`); let bigInteger: typeof import('https://cardstack.com/base/big-integer'); cardApi = await loader.import(`${baseRealm.url}card-api`); diff --git a/packages/host/tests/integration/search-index-test.gts b/packages/host/tests/integration/search-index-test.gts index 1b200eadda..63e8d13656 100644 --- a/packages/host/tests/integration/search-index-test.gts +++ b/packages/host/tests/integration/search-index-test.gts @@ -2643,17 +2643,17 @@ module('Integration | search-index', function (hooks) { // Exclude synthetic imports that encapsulate scoped CSS .filter((key) => !key.includes('glimmer-scoped.css')), [ - 'http://localhost:4201/base/card-api', - 'http://localhost:4201/base/contains-many-component', - 'http://localhost:4201/base/field-component', - 'http://localhost:4201/base/links-to-editor', - 'http://localhost:4201/base/links-to-many-component', - 'http://localhost:4201/base/number', - 'http://localhost:4201/base/shared-state', - 'http://localhost:4201/base/string', - 'http://localhost:4201/base/text-input-validator', - 'http://localhost:4201/base/watched-array', 'http://localhost:4202/test/person', + 'https://cardstack.com/base/card-api', + 'https://cardstack.com/base/contains-many-component', + 'https://cardstack.com/base/field-component', + 'https://cardstack.com/base/links-to-editor', + 'https://cardstack.com/base/links-to-many-component', + 'https://cardstack.com/base/number', + 'https://cardstack.com/base/shared-state', + 'https://cardstack.com/base/string', + 'https://cardstack.com/base/text-input-validator', + 'https://cardstack.com/base/watched-array', 'https://packages/@cardstack/boxel-ui/components', 'https://packages/@cardstack/boxel-ui/helpers', 'https://packages/@cardstack/boxel-ui/icons', diff --git a/packages/host/tests/unit/query-test.ts b/packages/host/tests/unit/query-test.ts index ed958ec8c4..e7e956b463 100644 --- a/packages/host/tests/unit/query-test.ts +++ b/packages/host/tests/unit/query-test.ts @@ -44,7 +44,10 @@ module('Unit | query', function (hooks) { hooks.beforeEach(async function () { let virtualNetwork = new VirtualNetwork(); loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(resolvedBaseRealmURL)); + virtualNetwork.addURLMapping( + new URL(baseRealm.url), + new URL(resolvedBaseRealmURL), + ); shimExternals(virtualNetwork); cardApi = await loader.import(`${baseRealm.url}card-api`); diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index 7e22e81d23..c93cd8b68c 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -138,7 +138,7 @@ let urlMappings = fromUrls.map((fromUrl, i) => [ new URL(String(toUrls[i]), `http://localhost:${port}`), ]); for (let [from, to] of urlMappings) { - loader.addURLMapping(from, to); + virtualNetwork.addURLMapping(from, to); } let hrefs = urlMappings.map(([from, to]) => [from.href, to.href]); let dist: string | URL; @@ -175,33 +175,33 @@ if (distURL) { let realmPermissions = getRealmPermissions(url); - realms.push( - new Realm( - { - url, - adapter: new NodeAdapter(resolve(String(path))), - loader, - indexRunner: getRunner, - runnerOptsMgr: manager, - getIndexHTML: async () => - readFileSync(join(distPath, 'index.html')).toString(), - matrix: { url: new URL(matrixURL), username, password }, - realmSecretSeed: REALM_SECRET_SEED, - permissions: realmPermissions.users, - }, - { - deferStartUp: true, - ...(useTestingDomain - ? { - useTestingDomain, - } - : {}), - }, - ), + let realm = new Realm( + { + url, + adapter: new NodeAdapter(resolve(String(path))), + loader, + indexRunner: getRunner, + runnerOptsMgr: manager, + getIndexHTML: async () => + readFileSync(join(distPath, 'index.html')).toString(), + matrix: { url: new URL(matrixURL), username, password }, + realmSecretSeed: REALM_SECRET_SEED, + permissions: realmPermissions.users, + }, + { + deferStartUp: true, + ...(useTestingDomain + ? { + useTestingDomain, + } + : {}), + }, ); + realms.push(realm); + virtualNetwork.mount(realm.maybeExternalHandle); } - let server = new RealmServer(realms, { + let server = new RealmServer(realms, virtualNetwork, { ...(distURL ? { assetsURL: new URL(distURL) } : {}), }); diff --git a/packages/realm-server/middleware/index.ts b/packages/realm-server/middleware/index.ts index 857b091674..786b0f2b33 100644 --- a/packages/realm-server/middleware/index.ts +++ b/packages/realm-server/middleware/index.ts @@ -1,10 +1,5 @@ import proxy from 'koa-proxies'; -import { - assetsDir, - boxelUIAssetsDir, - logger as getLogger, - type Realm, -} from '@cardstack/runtime-common'; +import { logger as getLogger } from '@cardstack/runtime-common'; import type Koa from 'koa'; import basicAuth from 'basic-auth'; @@ -94,56 +89,6 @@ export function ecsMetadata(ctxt: Koa.Context, next: Koa.Next) { return next(); } -export function assetRedirect( - assetsURL: URL, -): (ctxt: Koa.Context, next: Koa.Next) => void { - return (ctxt: Koa.Context, next: Koa.Next) => { - if (ctxt.path.startsWith(`/${assetsDir}`)) { - let redirectURL = new URL( - `./${ctxt.path.slice(assetsDir.length + 1)}`, - assetsURL, - ).href; - - if (redirectURL !== ctxt.href) { - ctxt.redirect(redirectURL); - return; - } - } - if (ctxt.path.startsWith(`/${boxelUIAssetsDir}`)) { - let redirectURL = new URL(`.${ctxt.path}`, assetsURL).href; - ctxt.redirect(redirectURL); - return; - } - return next(); - }; -} - -// requests for the root of the realm without a trailing slash aren't -// technically inside the realm (as the realm includes the trailing '/'). -// So issue a redirect in those scenarios. -export function rootRealmRedirect( - realms: Realm[], -): (ctxt: Koa.Context, next: Koa.Next) => void { - return (ctxt: Koa.Context, next: Koa.Next) => { - let url = fullRequestURL(ctxt); - - let realmUrlWithoutQueryParams = url.href.split('?')[0]; - if ( - !realmUrlWithoutQueryParams.endsWith('/') && - realms.find( - (r) => - r.loader.reverseResolution(`${realmUrlWithoutQueryParams}/`).href === - r.url, - ) - ) { - url.pathname = `${url.pathname}/`; - ctxt.redirect(`${url.href}`); // Adding a trailing slash to the URL one line above will update the href - return; - } - return next(); - }; -} - export function fullRequestURL(ctxt: Koa.Context): URL { let protocol = ctxt.req.headers['x-forwarded-proto'] === 'https' ? 'https' : 'http'; diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 76b1ba5e25..826ba2883c 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -8,6 +8,9 @@ import { assetsDir, logger, SupportedMimeType, + type VirtualNetwork, + boxelUIAssetsDir, + ResponseWithNodeStream, } from '@cardstack/runtime-common'; import { webStreamToText } from '@cardstack/runtime-common/stream'; import { setupCloseHandler } from './node-realm'; @@ -17,8 +20,6 @@ import { httpLogging, httpBasicAuth, ecsMetadata, - assetRedirect, - rootRealmRedirect, fullRequestURL, } from './middleware'; import convertAcceptHeaderQueryParam from './middleware/convert-accept-header-qp'; @@ -39,28 +40,54 @@ export class RealmServer { constructor( private realms: Realm[], + private virtualNetwork: VirtualNetwork, opts?: Options, ) { detectRealmCollision(realms); this.realms = realms; - // defaults to using the base realm to host assets (this is the dev env default) - // All realms should have URL mapping for the base realm - this.assetsURL = - opts?.assetsURL ?? - realms[0].loader.resolve(`${baseRealm.url}${assetsDir}`); + + this.assetsURL = opts?.assetsURL ?? new URL(`${baseRealm.url}${assetsDir}`); + + // TODO: Get rid of this redirect by providing the final absolute URL in the assetsURL. In dev, distURL should be localhost:4200, in prod, the deployed host app url. Remove distDir. + virtualNetwork.mount(async (request: Request) => { + let url = new URL(request.url); + let path = url.pathname; + + if (path.startsWith(`/${assetsDir}`)) { + let redirectURL = new URL( + `./${path.slice(assetsDir.length + 1)}`, + this.assetsURL, + ).href; + + if (redirectURL !== url.href) { + return new Response(null, { + status: 302, + headers: { + Location: redirectURL, + }, + }) as ResponseWithNodeStream; + } + } + + if (path.startsWith(`/${boxelUIAssetsDir}`)) { + let redirectURL = new URL(`.${path}`, this.assetsURL).href; + return new Response(null, { + status: 302, + headers: { + Location: redirectURL, + }, + }) as ResponseWithNodeStream; + } + + return null; + }); } @Memoize() get app() { let router = new Router(); router.head('/', livenessCheck); - router.get( - '/', - healthCheck, - this.serveIndex(), - rootRealmRedirect(this.realms), - this.serveFromRealm, - ); + router.get('/', healthCheck, this.serveIndex(), this.serveFromRealm); let app = new Koa() .use(httpLogging) @@ -90,10 +117,8 @@ export class RealmServer { await next(); }) .use(monacoMiddleware(this.assetsURL)) - .use(assetRedirect(this.assetsURL)) .use(convertAcceptHeaderQueryParam) .use(httpBasicAuth) - .use(rootRealmRedirect(this.realms)) .use(router.routes()) .use(this.serveFromRealm); @@ -130,49 +155,27 @@ export class RealmServer { if (ctxt.request.path === '/_boom') { throw new Error('boom'); } - - let realm = this.realms.find((r) => { - let reversedResolution = r.loader.reverseResolution( - fullRequestURL(ctxt).href, - ); - this.log.debug( - `Looking for realm to handle request with full URL: ${ - fullRequestURL(ctxt).href - } (reversed: ${reversedResolution.href})`, - ); - - let inRealm = r.paths.inRealm(reversedResolution); - this.log.debug( - `${reversedResolution} in realm ${JSON.stringify({ - url: r.url, - paths: r.paths, - })}: ${inRealm}`, - ); - return inRealm; - }); - - if (!realm) { - ctxt.status = 404; - return; - } - let reqBody: string | undefined; if (['POST', 'PATCH'].includes(ctxt.method)) { reqBody = await nodeStreamToText(ctxt.req); } - let reversedResolution = realm.loader.reverseResolution( - fullRequestURL(ctxt).href, - ); - - let request = new Request(reversedResolution.href, { + let url = fullRequestURL(ctxt).href; + let request = new Request(url, { method: ctxt.method, headers: ctxt.req.headers as { [name: string]: string }, ...(reqBody ? { body: reqBody } : {}), }); - setupCloseHandler(ctxt.res, request); - let realmResponse = await realm.handle(request); + let realmResponse = await this.virtualNetwork.handle( + request, + (mappedRequest) => { + // Setup this handler only after the request has been mapped because + // the *mapped request* is the one that gets closed, not the original one + setupCloseHandler(ctxt.res, mappedRequest); + }, + ); + let { status, statusText, headers, body, nodeStream } = realmResponse; ctxt.status = status; ctxt.message = statusText; @@ -180,7 +183,7 @@ export class RealmServer { ctxt.set(header, value); } if (!headers.get('content-type')) { - let fileName = reversedResolution.href.split('/').pop()!; + let fileName = url.split('/').pop()!; ctxt.type = mime.lookup(fileName) || 'application/octet-stream'; } diff --git a/packages/realm-server/tests/helpers/index.ts b/packages/realm-server/tests/helpers/index.ts index 1beaa2dfd5..66d25e82f8 100644 --- a/packages/realm-server/tests/helpers/index.ts +++ b/packages/realm-server/tests/helpers/index.ts @@ -7,6 +7,7 @@ import { Loader, baseRealm, RealmPermissions, + VirtualNetwork, } from '@cardstack/runtime-common'; import { makeFastBootIndexRunner } from '../../fastboot'; import { RunnerOptionsManager } from '@cardstack/runtime-common/search-index'; @@ -66,10 +67,14 @@ export async function createRealm( }); } -export function setupBaseRealmServer(hooks: NestedHooks, loader: Loader) { +export function setupBaseRealmServer( + hooks: NestedHooks, + loader: Loader, + virtualNetwork: VirtualNetwork, +) { let baseRealmServer: Server; hooks.before(async function () { - baseRealmServer = await runBaseRealmServer(loader); + baseRealmServer = await runBaseRealmServer(loader, virtualNetwork); }); hooks.after(function () { @@ -77,9 +82,12 @@ export function setupBaseRealmServer(hooks: NestedHooks, loader: Loader) { }); } -export async function runBaseRealmServer(loader: Loader) { +export async function runBaseRealmServer( + loader: Loader, + virtualNetwork: VirtualNetwork, +) { let localBaseRealmURL = new URL(localBaseRealm); - loader.addURLMapping(new URL(baseRealm.url), localBaseRealmURL); + virtualNetwork.addURLMapping(new URL(baseRealm.url), localBaseRealmURL); let testBaseRealm = await createRealm( loader, @@ -87,13 +95,15 @@ export async function runBaseRealmServer(loader: Loader) { undefined, baseRealm.url, ); + virtualNetwork.mount(testBaseRealm.maybeExternalHandle); await testBaseRealm.ready; - let testBaseRealmServer = new RealmServer([testBaseRealm]); + let testBaseRealmServer = new RealmServer([testBaseRealm], virtualNetwork); return testBaseRealmServer.listen(parseInt(localBaseRealmURL.port)); } export async function runTestRealmServer( loader: Loader, + virtualNetwork: VirtualNetwork, dir: string, flatFiles: Record = {}, testRealmURL: URL, @@ -106,10 +116,12 @@ export async function runTestRealmServer( testRealmURL.href, permissions, ); + virtualNetwork.mount(testRealm.maybeExternalHandle); await testRealm.ready; - let testRealmServer = await new RealmServer([testRealm]).listen( - parseInt(testRealmURL.port), - ); + let testRealmServer = await new RealmServer( + [testRealm], + virtualNetwork, + ).listen(parseInt(testRealmURL.port)); return { testRealm, testRealmServer, diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index 69c5751e24..404187b1cf 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -5,4 +5,5 @@ import './indexing-test'; import './module-syntax-test'; import './permissions/permission-checker-test'; import './auth-client-test'; +import './virtual-network-test'; import './pg-test'; diff --git a/packages/realm-server/tests/indexing-test.ts b/packages/realm-server/tests/indexing-test.ts index f748285409..5b2e99c9aa 100644 --- a/packages/realm-server/tests/indexing-test.ts +++ b/packages/realm-server/tests/indexing-test.ts @@ -8,6 +8,7 @@ import { } from '@cardstack/runtime-common'; import { createRealm, + localBaseRealm, testRealm, setupCardLogs, setupBaseRealmServer, @@ -36,10 +37,7 @@ module('indexing', function (hooks) { let virtualNetwork = new VirtualNetwork(); let loader = virtualNetwork.createLoader(); - loader.addURLMapping( - new URL(baseRealm.url), - new URL('http://localhost:4201/base/'), - ); + virtualNetwork.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); shimExternals(virtualNetwork); setupCardLogs( @@ -50,15 +48,10 @@ module('indexing', function (hooks) { let dir: string; let realm: Realm; - setupBaseRealmServer(hooks, loader); + setupBaseRealmServer(hooks, loader, virtualNetwork); hooks.beforeEach(async function () { let testRealmLoader = virtualNetwork.createLoader(); - testRealmLoader.addURLMapping( - new URL(baseRealm.url), - new URL('http://localhost:4201/base/'), - ); - shimExternals(virtualNetwork); dir = dirSync().name; realm = await createRealm(testRealmLoader, dir, { diff --git a/packages/realm-server/tests/loader-test.ts b/packages/realm-server/tests/loader-test.ts index 8746746cd1..f16b8c9acf 100644 --- a/packages/realm-server/tests/loader-test.ts +++ b/packages/realm-server/tests/loader-test.ts @@ -4,11 +4,9 @@ import { dirSync, setGracefulCleanup, DirResult } from 'tmp'; import { createRealm, setupBaseRealmServer, - localBaseRealm, runTestRealmServer, } from './helpers'; import { copySync } from 'fs-extra'; -import { baseRealm } from '@cardstack/runtime-common'; import { shimExternals } from '../lib/externals'; import { Server } from 'http'; import { join } from 'path'; @@ -25,20 +23,22 @@ module('loader', function (hooks) { let virtualNetwork = new VirtualNetwork(); let loader = virtualNetwork.createLoader(); - loader.addURLMapping( - new URL(baseRealm.url), - new URL('http://localhost:4201/base/'), - ); shimExternals(virtualNetwork); - setupBaseRealmServer(hooks, loader); + setupBaseRealmServer(hooks, loader, virtualNetwork); hooks.beforeEach(async function () { dir = dirSync(); copySync(join(__dirname, 'cards'), dir.name); testRealmServer = ( - await runTestRealmServer(loader, dir.name, undefined, testRealmURL) + await runTestRealmServer( + loader, + virtualNetwork, + dir.name, + undefined, + testRealmURL, + ) ).testRealmServer; }); @@ -48,7 +48,6 @@ module('loader', function (hooks) { test('can dynamically load modules with cycles', async function (assert) { let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); let module = await loader.import<{ three(): number }>( `${testRealmHref}cycle-two`, ); @@ -57,7 +56,6 @@ module('loader', function (hooks) { test('can resolve multiple import load races against a common dep', async function (assert) { let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); let a = loader.import<{ a(): string }>(`${testRealmHref}a`); let b = loader.import<{ b(): string }>(`${testRealmHref}b`); let [aModule, bModule] = await Promise.all([a, b]); @@ -67,7 +65,6 @@ module('loader', function (hooks) { test('can resolve a import deadlock', async function (assert) { let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); let a = loader.import<{ a(): string }>(`${testRealmHref}deadlock/a`); let b = loader.import<{ b(): string }>(`${testRealmHref}deadlock/b`); let c = loader.import<{ c(): string }>(`${testRealmHref}deadlock/c`); @@ -91,7 +88,6 @@ module('loader', function (hooks) { 'http://example.com/', ); loader.registerURLHandler(realm.maybeHandle.bind(realm)); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); await realm.ready; let { checkImportMeta, myLoader } = await loader.import<{ @@ -114,7 +110,6 @@ module('loader', function (hooks) { test('can determine consumed modules when an error is encountered during loading', async function (assert) { let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); try { await loader.import<{ d(): string }>(`${testRealmHref}d`); throw new Error(`expected error was not thrown`); @@ -132,7 +127,6 @@ module('loader', function (hooks) { test('can get consumed modules within a cycle', async function (assert) { let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); await loader.import<{ three(): number }>(`${testRealmHref}cycle-two`); let modules = await loader.getConsumedModules(`${testRealmHref}cycle-two`); assert.deepEqual(modules, [ @@ -144,7 +138,6 @@ module('loader', function (hooks) { test('supports identify API', async function (assert) { let loader = virtualNetwork.createLoader(); shimExternals(virtualNetwork); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); let { Person } = await loader.import<{ Person: unknown }>( `${testRealmHref}person`, ); @@ -162,7 +155,6 @@ module('loader', function (hooks) { test('exports cannot be mutated', async function (assert) { let loader = virtualNetwork.createLoader(); shimExternals(virtualNetwork); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); let module = await loader.import<{ Person: unknown }>( `${testRealmHref}person`, ); @@ -174,7 +166,6 @@ module('loader', function (hooks) { test('can get a loader used to import a specific card', async function (assert) { let loader = virtualNetwork.createLoader(); shimExternals(virtualNetwork); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); let module = await loader.import(`${testRealmHref}person`); let card = module.Person; let testingLoader = Loader.getLoaderFor(card); diff --git a/packages/realm-server/tests/module-syntax-test.ts b/packages/realm-server/tests/module-syntax-test.ts index 9bc87cbb73..9af7327424 100644 --- a/packages/realm-server/tests/module-syntax-test.ts +++ b/packages/realm-server/tests/module-syntax-test.ts @@ -1,26 +1,11 @@ import { module, test } from 'qunit'; import { ModuleSyntax } from '@cardstack/runtime-common/module-syntax'; -import { dirSync } from 'tmp'; -import { - baseRealm, - baseCardRef, - baseFieldRef, - VirtualNetwork, -} from '@cardstack/runtime-common'; -import { testRealm, createRealm } from './helpers'; + +import { baseCardRef, baseFieldRef } from '@cardstack/runtime-common'; +import { testRealm } from './helpers'; import '@cardstack/runtime-common/helpers/code-equality-assertion'; -import { shimExternals } from '../lib/externals'; module('module-syntax', function () { - let virtualNetwork = new VirtualNetwork(); - let loader = virtualNetwork.createLoader(); - - loader.addURLMapping( - new URL(baseRealm.url), - new URL('http://localhost:4201/base/'), - ); - shimExternals(virtualNetwork); - function addField(src: string, addFieldAtIndex?: number) { let mod = new ModuleSyntax(src, new URL(`${testRealm}dir/person.gts`)); mod.addField({ @@ -758,17 +743,6 @@ module('module-syntax', function () { }); test('can add a linksTo field', async function (assert) { - let realm = await createRealm(loader, dirSync().name, { - 'pet.gts': ` - import { contains, field, CardDef } from "https://cardstack.com/base/card-api"; - import StringField from "https://cardstack.com/base/string"; - export class Pet extends CardDef { - @field petName = contains(StringField); - } - `, - }); - await realm.ready; - let src = ` import { contains, field, CardDef } from "https://cardstack.com/base/card-api"; import StringField from "https://cardstack.com/base/string"; diff --git a/packages/realm-server/tests/realm-server-test.ts b/packages/realm-server/tests/realm-server-test.ts index 205e9f194e..d0380cd59a 100644 --- a/packages/realm-server/tests/realm-server-test.ts +++ b/packages/realm-server/tests/realm-server-test.ts @@ -30,10 +30,10 @@ import { import { stringify } from 'qs'; import { Query } from '@cardstack/runtime-common/query'; import { - localBaseRealm, setupCardLogs, setupBaseRealmServer, runTestRealmServer, + localBaseRealm, } from './helpers'; import '@cardstack/runtime-common/helpers/code-equality-assertion'; import eventSource from 'eventsource'; @@ -118,7 +118,6 @@ module('Realm Server', function (hooks) { let virtualNetwork = new VirtualNetwork(); let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); shimExternals(virtualNetwork); setupCardLogs( @@ -126,7 +125,7 @@ module('Realm Server', function (hooks) { async () => await loader.import(`${baseRealm.url}card-api`), ); - setupBaseRealmServer(hooks, loader); + setupBaseRealmServer(hooks, loader, virtualNetwork); hooks.beforeEach(async function () { dir = dirSync(); @@ -1714,18 +1713,14 @@ module('Realm Server', function (hooks) { '*': ['read', 'write'], })); - let virtualNetwork = new VirtualNetwork(); - let testRealmServer2Loader = virtualNetwork.createLoader(); - testRealmServer2Loader.addURLMapping( - new URL(baseRealm.url), - new URL(localBaseRealm), - ); + shimExternals(virtualNetwork); testRealmServer2 = ( await runTestRealmServer( testRealmServer2Loader, + virtualNetwork, dir.name, undefined, testRealm2URL, @@ -2032,7 +2027,6 @@ module('Realm Server serving from root', function (hooks) { let virtualNetwork = new VirtualNetwork(); let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); shimExternals(virtualNetwork); setupCardLogs( @@ -2040,23 +2034,18 @@ module('Realm Server serving from root', function (hooks) { async () => await loader.import(`${baseRealm.url}card-api`), ); - setupBaseRealmServer(hooks, loader); + setupBaseRealmServer(hooks, loader, virtualNetwork); hooks.beforeEach(async function () { dir = dirSync(); copySync(join(__dirname, 'cards'), dir.name); - let virtualNetwork = new VirtualNetwork(); - let testRealmServerLoader = virtualNetwork.createLoader(); - testRealmServerLoader.addURLMapping( - new URL(baseRealm.url), - new URL(localBaseRealm), - ); testRealmServer = ( await runTestRealmServer( testRealmServerLoader, + virtualNetwork, dir.name, undefined, testRealmURL, @@ -2236,7 +2225,6 @@ module('Realm Server serving from a subdirectory', function (hooks) { let virtualNetwork = new VirtualNetwork(); let loader = virtualNetwork.createLoader(); - loader.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); shimExternals(virtualNetwork); setupCardLogs( @@ -2244,21 +2232,18 @@ module('Realm Server serving from a subdirectory', function (hooks) { async () => await loader.import(`${baseRealm.url}card-api`), ); - setupBaseRealmServer(hooks, loader); + setupBaseRealmServer(hooks, loader, virtualNetwork); hooks.beforeEach(async function () { dir = dirSync(); copySync(join(__dirname, 'cards'), dir.name); let testRealmServerLoader = virtualNetwork.createLoader(); - testRealmServerLoader.addURLMapping( - new URL(baseRealm.url), - new URL(localBaseRealm), - ); testRealmServer = ( await runTestRealmServer( testRealmServerLoader, + virtualNetwork, dir.name, undefined, new URL('http://127.0.0.1:4446/demo/'), @@ -2303,16 +2288,14 @@ async function setupPermissionedRealm(permissions: RealmPermissions) { let dir = dirSync(); copySync(join(__dirname, 'cards'), dir.name); let virtualNetwork = new VirtualNetwork(); - let testRealmServerLoader = virtualNetwork.createLoader(); - testRealmServerLoader.addURLMapping( - new URL(baseRealm.url), - new URL(localBaseRealm), - ); - shimExternals(virtualNetwork); + virtualNetwork.addURLMapping(new URL(baseRealm.url), new URL(localBaseRealm)); + + let testRealmServerLoader = virtualNetwork.createLoader(); ({ testRealm, testRealmServer } = await runTestRealmServer( testRealmServerLoader, + virtualNetwork, dir.name, undefined, testRealmURL, diff --git a/packages/realm-server/tests/virtual-network-test.ts b/packages/realm-server/tests/virtual-network-test.ts new file mode 100644 index 0000000000..df9c06f1ac --- /dev/null +++ b/packages/realm-server/tests/virtual-network-test.ts @@ -0,0 +1,34 @@ +import { + ResponseWithNodeStream, + VirtualNetwork, +} from '@cardstack/runtime-common'; +import { module, test } from 'qunit'; + +module('virtual-network', function () { + test('will respond wilth real (not virtual) url when handler makes a redirect', async function (assert) { + let virtualNetwork = new VirtualNetwork(); + virtualNetwork.addURLMapping( + new URL('https://cardstack.com/base/'), + new URL('http://localhost:4201/base/'), + ); + virtualNetwork.mount(async (_request: Request) => { + // Normally there would be some redirection logic here, but for this test we just want to make sure that the redirect is handled correctly + return new Response(null, { + status: 302, + headers: { + Location: 'https://cardstack.com/base/__boxel/assets/', // This virtual url should be converted to a real url so that the client can follow the redirect + }, + }) as ResponseWithNodeStream; + }); + + let response = await virtualNetwork.handle( + new Request('http://localhost:4201/__boxel/assets/'), + ); + + assert.strictEqual(response.status, 302); + assert.strictEqual( + response.headers.get('Location'), + 'http://localhost:4201/base/__boxel/assets/', + ); + }); +}); diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 2969683af9..7e3a7e767e 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -58,13 +58,12 @@ export const isNode = export { Realm } from './realm'; export { SupportedMimeType } from './router'; -export { VirtualNetwork } from './virtual-network'; +export { VirtualNetwork, type ResponseWithNodeStream } from './virtual-network'; export type { Kind, RealmAdapter, FileRef, - ResponseWithNodeStream, RealmInfo, TokenClaims, RealmPermissions, diff --git a/packages/runtime-common/loader.ts b/packages/runtime-common/loader.ts index 6337a1b4b0..8881728cf1 100644 --- a/packages/runtime-common/loader.ts +++ b/packages/runtime-common/loader.ts @@ -2,28 +2,12 @@ import TransformModulesAmdPlugin from 'transform-modules-amd-plugin'; import { transformSync } from '@babel/core'; import { Deferred } from './deferred'; import { trimExecutableExtension, logger } from './index'; -import { RealmPaths } from './paths'; + import { CardError } from './error'; import flatMap from 'lodash/flatMap'; import { decodeScopedCSSRequest, isScopedCSSRequest } from 'glimmer-scoped-css'; import jsEscapeString from 'js-string-escape'; -// this represents a URL that has already been resolved to aid in documenting -// when resolution has already been performed -export interface ResolvedURL extends URL { - _isResolved: undefined; -} - -function isResolvedURL(url: URL | ResolvedURL): url is ResolvedURL { - return '_isResolved' in url; -} - -export function makeResolvedURL(unresolvedURL: URL | string): ResolvedURL { - let resolvedURL = new URL(unresolvedURL) as ResolvedURL; - resolvedURL._isResolved = undefined; - return resolvedURL; -} - type FetchingModule = { state: 'fetching'; deferred: Deferred; @@ -88,18 +72,18 @@ type EvaluatableModule = | BrokenModule; type UnregisteredDep = - | { type: 'dep'; moduleURL: ResolvedURL } + | { type: 'dep'; moduleURL: URL } | { type: '__import_meta__' } | { type: 'exports' }; type EvaluatableDep = | { type: 'dep'; - moduleURL: ResolvedURL; + moduleURL: URL; } | { type: 'completing-dep'; - moduleURL: ResolvedURL; + moduleURL: URL; } | { type: '__import_meta__' } | { type: 'exports' }; @@ -115,13 +99,6 @@ export class Loader { private modules = new Map(); private urlHandlers: RequestHandler[] = [maybeHandleScopedCSSRequest]; - // use a tuple array instead of a map so that we can support reversing - // different resolutions back to the same URL. the resolution that we apply - // will be in order of precedence. consider 2 realms in the same server - // wanting to communicate via localhost resolved URL's, but also a browser - // that talks to the realm (we need to reverse the resolution in the server.ts - // to figure out which realm the request is talking to) - private urlMappings: [string, string][] = []; private moduleShims = new Map>(); private identities = new WeakMap< Function, @@ -145,17 +122,12 @@ export class Loader { static cloneLoader(loader: Loader): Loader { let clone = new Loader(loader.fetchImplementation, loader.resolveImport); clone.urlHandlers = loader.urlHandlers; - clone.urlMappings = loader.urlMappings; for (let [moduleIdentifier, module] of loader.moduleShims) { clone.shimModule(moduleIdentifier, module); } return clone; } - addURLMapping(from: URL, to: URL) { - this.urlMappings.push([from.href, to.href]); - } - registerURLHandler(handler: RequestHandler) { this.urlHandlers.push(handler); } @@ -193,7 +165,7 @@ export class Loader { } consumed.add(moduleIdentifier); - let resolvedModuleIdentifier = this.resolve(new URL(moduleIdentifier)); + let resolvedModuleIdentifier = new URL(moduleIdentifier); let module = this.getModule(resolvedModuleIdentifier.href); if (!module || module.state === 'fetching') { @@ -258,7 +230,7 @@ export class Loader { async import(moduleIdentifier: string): Promise { moduleIdentifier = this.resolveImport(moduleIdentifier); - let resolvedModule = this.resolve(moduleIdentifier); + let resolvedModule = new URL(moduleIdentifier); let resolvedModuleIdentifier = resolvedModule.href; await this.advanceToState(resolvedModule, 'evaluated'); @@ -277,7 +249,7 @@ export class Loader { } private async advanceToState( - resolvedURL: ResolvedURL, + resolvedURL: URL, targetState: | 'registered-completing-deps' | 'registered-with-deps' @@ -448,39 +420,14 @@ export class Loader { } } - private asUnresolvedRequest( + private asRequest( urlOrRequest: string | URL | Request, init?: RequestInit, ): Request { if (urlOrRequest instanceof Request) { return urlOrRequest; } else { - let unresolvedURL = - typeof urlOrRequest === 'string' - ? new URL(urlOrRequest) - : isResolvedURL(urlOrRequest) - ? this.reverseResolution(urlOrRequest) - : urlOrRequest; - return new Request(unresolvedURL.href, init); - } - } - - private asResolvedRequest( - urlOrRequest: string | URL | Request, - init?: RequestInit, - ): Request { - if (urlOrRequest instanceof Request) { - return new Request(this.resolve(urlOrRequest.url).href, { - method: urlOrRequest.method, - headers: urlOrRequest.headers, - body: urlOrRequest.body, - }); - } else if (typeof urlOrRequest === 'string') { - return new Request(this.resolve(urlOrRequest), init); - } else if (isResolvedURL(urlOrRequest)) { return new Request(urlOrRequest, init); - } else { - return new Request(this.resolve(urlOrRequest), init); } } @@ -526,7 +473,7 @@ export class Loader { ): Promise { try { for (let handler of this.urlHandlers) { - let request = this.asUnresolvedRequest(urlOrRequest, init); + let request = this.asRequest(urlOrRequest, init); let result = await handler(request); if (result) { @@ -535,7 +482,7 @@ export class Loader { } let shimmedModule = this.moduleShims.get( - this.asUnresolvedRequest(urlOrRequest, init).url, + this.asRequest(urlOrRequest, init).url, ); if (shimmedModule) { let response = new Response(); @@ -543,9 +490,7 @@ export class Loader { return response; } - return await this.fetchImplementation( - this.asResolvedRequest(urlOrRequest, init), - ); + return await this.fetchImplementation(this.asRequest(urlOrRequest, init)); } catch (err: any) { let url = urlOrRequest instanceof Request @@ -559,49 +504,6 @@ export class Loader { } } - resolve(moduleIdentifier: string | URL, relativeTo?: URL): ResolvedURL { - let absoluteURL = new URL(moduleIdentifier, relativeTo); - for (let [sourceURL, to] of this.urlMappings) { - let sourcePath = new RealmPaths(new URL(sourceURL)); - if (sourcePath.inRealm(absoluteURL)) { - let toPath = new RealmPaths(new URL(to)); - if (absoluteURL.href.endsWith('/')) { - return makeResolvedURL( - toPath.directoryURL(sourcePath.local(absoluteURL)), - ); - } else { - return makeResolvedURL( - toPath.fileURL( - sourcePath.local(absoluteURL, { preserveQuerystring: true }), - ), - ); - } - } - } - return makeResolvedURL(absoluteURL); - } - - reverseResolution( - moduleIdentifier: string | ResolvedURL, - relativeTo?: URL, - ): URL { - let absoluteURL = new URL(moduleIdentifier, relativeTo); - for (let [sourceURL, to] of this.urlMappings) { - let sourcePath = new RealmPaths(new URL(sourceURL)); - let destinationPath = new RealmPaths(to); - if (destinationPath.inRealm(absoluteURL)) { - if (absoluteURL.href.endsWith('/')) { - return sourcePath.directoryURL(destinationPath.local(absoluteURL)); - } else { - return sourcePath.fileURL( - destinationPath.local(absoluteURL, { preserveQuerystring: true }), - ); - } - } - } - return absoluteURL; - } - private getModule(moduleIdentifier: string): Module | undefined { return this.modules.get(trimModuleIdentifier(moduleIdentifier)); } @@ -611,9 +513,7 @@ export class Loader { } private createModuleProxy(module: any, moduleIdentifier: string) { - let moduleId = trimExecutableExtension( - this.reverseResolution(moduleIdentifier), - ).href; + let moduleId = trimExecutableExtension(new URL(moduleIdentifier)).href; return new Proxy(module, { get: (target, property, received) => { let value = Reflect.get(target, property, received); @@ -634,7 +534,7 @@ export class Loader { }); } - private async fetchModule(moduleURL: ResolvedURL): Promise { + private async fetchModule(moduleURL: URL): Promise { let moduleIdentifier = typeof moduleURL === 'string' ? moduleURL : moduleURL.href; @@ -703,7 +603,7 @@ export class Loader { } else { return { type: 'dep', - moduleURL: this.resolve( + moduleURL: new URL( this.resolveImport(depId), new URL(moduleIdentifier), ), @@ -800,7 +700,7 @@ export class Loader { } private async load( - moduleURL: ResolvedURL, + moduleURL: URL, ): Promise< | { type: 'source'; source: string } | { type: 'shimmed'; module: Record } diff --git a/packages/runtime-common/package-shim-handler.ts b/packages/runtime-common/package-shim-handler.ts new file mode 100644 index 0000000000..49c10afe7b --- /dev/null +++ b/packages/runtime-common/package-shim-handler.ts @@ -0,0 +1,52 @@ +import { logger, trimExecutableExtension } from './index'; + +function trimModuleIdentifier(moduleIdentifier: string): string { + return trimExecutableExtension(new URL(moduleIdentifier)).href; +} + +export const PACKAGES_FAKE_ORIGIN = 'https://packages/'; + +export class PackageShimHandler { + private resolveImport: (moduleIdentifier: string) => string; + private modules = new Map>(); + private log = logger('shim-handler'); + + constructor(resolveImport: (moduleIdentifier: string) => string) { + this.resolveImport = resolveImport; + } + + handle = async (request: Request): Promise => { + if (request.url.startsWith(PACKAGES_FAKE_ORIGIN)) { + try { + let shimmedModule = this.getModule(request.url); + if (shimmedModule) { + let response = new Response(); + (response as any)[Symbol.for('shimmed-module')] = shimmedModule; + return response; + } + + return null; + } catch (err: any) { + this.log.error( + `PackageShimHandler#handle threw an error handling ${request.url}`, + err, + ); + return null; + } + } + return null; + }; + + shimModule(moduleIdentifier: string, module: Record) { + moduleIdentifier = this.resolveImport(moduleIdentifier); + this.setModule(moduleIdentifier, module); + } + + private setModule(moduleIdentifier: string, module: Record) { + this.modules.set(trimModuleIdentifier(moduleIdentifier), module); + } + + private getModule(moduleIdentifier: string): Record | undefined { + return this.modules.get(trimModuleIdentifier(moduleIdentifier)); + } +} diff --git a/packages/runtime-common/paths.ts b/packages/runtime-common/paths.ts index 37b9839f51..76eeed25ee 100644 --- a/packages/runtime-common/paths.ts +++ b/packages/runtime-common/paths.ts @@ -17,7 +17,7 @@ export class RealmPaths { url = new URL(url); } - if (!url.href.startsWith(this.url)) { + if (!this.inRealm(url)) { let error = new Error(`realm ${this.url} does not contain ${url.href}`); (error as any).status = 404; throw error; @@ -52,7 +52,10 @@ export class RealmPaths { } inRealm(url: URL): boolean { - return url.href.startsWith(this.url); + return ( + url.href.startsWith(this.url) || + url.href.split('?')[0] == this.url.replace(/\/$/, '') // check if url without querystring same as realm url without trailing slash (for detecting root realm urls with missing trailing slash) + ); } } diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 0a66d33044..da4eb54772 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -72,7 +72,6 @@ import { lookupRouteTable, } from './router'; import { parseQueryString } from './query'; -//@ts-ignore service worker can't handle this import type { Readable } from 'stream'; import { type CardDef } from 'https://cardstack.com/base/card-api'; import type * as CardAPI from 'https://cardstack.com/base/card-api'; @@ -85,6 +84,7 @@ import { Sha256 } from '@aws-crypto/sha256-js'; import { JsonWebTokenError, TokenExpiredError } from 'jsonwebtoken'; import RealmPermissionChecker from './realm-permission-checker'; +import type { ResponseWithNodeStream } from './virtual-network'; export type RealmInfo = { name: string; @@ -98,10 +98,6 @@ export interface FileRef { lastModified: number; } -export interface ResponseWithNodeStream extends Response { - nodeStream?: Readable; -} - export interface TokenClaims { user: string; realm: string; @@ -587,12 +583,25 @@ export class Realm { return this.#startedUp.promise; } - async maybeHandle(request: Request): Promise { + maybeHandle = async ( + request: Request, + ): Promise => { if (!this.paths.inRealm(new URL(request.url))) { return null; } return await this.internalHandle(request, true); - } + }; + + // This is scaffolding that should be deleted once we can finish the isolated + // loader refactor + maybeExternalHandle = async ( + request: Request, + ): Promise => { + if (!this.paths.inRealm(new URL(request.url))) { + return null; + } + return await this.internalHandle(request, false); + }; async handle(request: Request): Promise { return this.internalHandle(request, false); @@ -790,6 +799,11 @@ export class Realm { request: Request, isLocal: boolean, ): Promise { + let redirectResponse = this.rootRealmRedirect(request); + if (redirectResponse) { + return redirectResponse; + } + try { // local requests are allowed to query the realm as the index is being built up if (!isLocal) { @@ -825,6 +839,25 @@ export class Realm { } } + // Requests for the root of the realm without a trailing slash aren't + // technically inside the realm (as the realm includes the trailing '/'), + // so issue a redirect in those scenarios. + private rootRealmRedirect(request: Request) { + let url = new URL(request.url); + let urlWithoutQueryParams = url.protocol + '//' + url.host + url.pathname; + if (`${urlWithoutQueryParams}/` === this.url) { + return new Response(null, { + status: 302, + headers: { + Location: String(url.searchParams) + ? `${this.url}?${url.searchParams}` + : this.url, + }, + }); + } + return undefined; + } + async fallbackHandle(request: Request) { let url = new URL(request.url); let localPath = this.paths.local(url); @@ -852,17 +885,13 @@ export class Realm { } async getIndexHTML(opts?: IndexHTMLOptions): Promise { - let resolvedBaseRealmURL = this.#searchIndex.loader.resolve( - baseRealm.url, - ).href; let indexHTML = (await this.#getIndexHTML()).replace( /()/, (_match, g1, g2, g3) => { let config = JSON.parse(decodeURIComponent(g2)); config = merge({}, config, { ownRealmURL: this.url, // unresolved url - resolvedBaseRealmURL, - resolvedOwnRealmURL: this.#searchIndex.loader.resolve(this.url).href, + resolvedOwnRealmURL: this.url, hostsOwnAssets: !isNode, realmsServed: opts?.realmsServed, }); @@ -971,7 +1000,7 @@ export class Realm { request: Request, neededPermission: 'read' | 'write', ) { - let endpontsWithoutAuthNeeded: RouteTable = new Map([ + let endpointsWithoutAuthNeeded: RouteTable = new Map([ // authentication endpoint [ SupportedMimeType.Session, @@ -990,7 +1019,7 @@ export class Realm { ]); if ( - lookupRouteTable(endpontsWithoutAuthNeeded, this.paths, request) || + lookupRouteTable(endpointsWithoutAuthNeeded, this.paths, request) || request.method === 'HEAD' || // If the realm is public readable or writable, do not require a JWT (neededPermission === 'read' && diff --git a/packages/runtime-common/virtual-network.ts b/packages/runtime-common/virtual-network.ts index 8b57394d45..353fc05a94 100644 --- a/packages/runtime-common/virtual-network.ts +++ b/packages/runtime-common/virtual-network.ts @@ -1,8 +1,16 @@ +import { RealmPaths } from './paths'; import { Loader } from './loader'; import type { RunnerOpts } from './search-index'; +import { + PackageShimHandler, + PACKAGES_FAKE_ORIGIN, +} from './package-shim-handler'; +import type { Readable } from 'stream'; +export interface ResponseWithNodeStream extends Response { + nodeStream?: Readable; +} const isFastBoot = typeof (globalThis as any).FastBoot !== 'undefined'; -const PACKAGES_FAKE_ORIGIN = 'https://packages/'; function getNativeFetch(): typeof fetch { if (isFastBoot) { @@ -19,11 +27,11 @@ function getNativeFetch(): typeof fetch { } } -export type Handler = (req: Request) => Promise; +export type Handler = (req: Request) => Promise; export class VirtualNetwork { - private nativeFetch = getNativeFetch(); private handlers: Handler[] = []; + private urlMappings: [string, string][] = []; private resolveImport = (moduleIdentifier: string) => { if (!isUrlLike(moduleIdentifier)) { @@ -32,18 +40,10 @@ export class VirtualNetwork { return moduleIdentifier; }; - private shimmingLoader = new Loader(() => { - throw new Error('This loader should never call fetch'); - }, this.resolveImport); + private packageShimHandler = new PackageShimHandler(this.resolveImport); constructor() { - this.mount(async (request) => { - if (request.url.startsWith(PACKAGES_FAKE_ORIGIN)) { - return this.shimmingLoader.fetch(request); - } - - return null; - }); + this.mount(this.packageShimHandler.handle); } createLoader() { @@ -51,7 +51,49 @@ export class VirtualNetwork { } shimModule(moduleIdentifier: string, module: Record) { - this.shimmingLoader.shimModule(moduleIdentifier, module); + this.packageShimHandler.shimModule(moduleIdentifier, module); + } + + addURLMapping(from: URL, to: URL) { + this.urlMappings.push([from.href, to.href]); + } + + private nativeFetch(...args: Parameters) { + return getNativeFetch()(...args); + } + + private resolveURLMapping( + url: string, + direction: 'virtual-to-real' | 'real-to-virtual', + ): string | undefined { + let absoluteURL = new URL(url); + for (let [virtual, real] of this.urlMappings) { + let sourcePath = new RealmPaths( + new URL(direction === 'virtual-to-real' ? virtual : real), + ); + if (sourcePath.inRealm(absoluteURL)) { + let toPath = new RealmPaths( + new URL(direction === 'virtual-to-real' ? real : virtual), + ); + if (absoluteURL.href.endsWith('/')) { + return toPath.directoryURL(sourcePath.local(absoluteURL)).href; + } else { + let local = sourcePath.local(absoluteURL, { + preserveQuerystring: true, + }); + let resolved = toPath.fileURL(local).href; + + // A special case for root realm urls with missing trailing slash, for + // example http://localhost:4201/base – we want the mapped url also not to have a trailing slash + // (so that the realm handler knows it needs to redirect to the correct url with a trailing slash) + if (local === '' && !absoluteURL.pathname.endsWith('/')) { + resolved = resolved.replace(/\/$/, ''); + } + return resolved; + } + } + } + return undefined; } mount(handler: Handler) { @@ -67,6 +109,78 @@ export class VirtualNetwork { ? urlOrRequest : new Request(urlOrRequest, init); + let internalRequest = await this.mapRequest(request, 'virtual-to-real'); + let response = await this.runFetch(internalRequest, init); + if (internalRequest !== request) { + Object.defineProperty(response, 'url', { + value: + this.resolveURLMapping(response.url, 'real-to-virtual') ?? + response.url, + }); + } + return response; + }; + + // This method is used to handle the boundary between the real and virtual network, + // when a request is made to the realm from the realm server - it maps requests + // by changing their URL from real to virtual, as defined in the url mapping config + // (e.g http://localhost:4201/base to https://cardstack.com/base) so that the realms + // that have a virtual URL know that they are being requested + async handle( + request: Request, + onMappedRequest?: (request: Request) => void, + ): Promise { + let internalRequest = await this.mapRequest(request, 'real-to-virtual'); + if (onMappedRequest) { + onMappedRequest(internalRequest); + } + + for (let handler of this.handlers) { + let response = await handler(internalRequest); + if (response) { + this.mapRedirectionURL(response); + return response; + } + } + return new Response(undefined, { status: 404 }); + } + + private async mapRequest( + request: Request, + direction: 'virtual-to-real' | 'real-to-virtual', + ) { + let remappedUrl = this.resolveURLMapping(request.url, direction); + + if (remappedUrl) { + return await buildRequest(remappedUrl, request); + } else { + return request; + } + } + + private mapRedirectionURL(response: Response): void { + if (response.status > 300 && response.status < 400) { + let redirectionURL = response.headers.get('Location')!; + let isRelativeRedirectionURL = !/^[a-z][a-z0-9+.-]*:|\/\//i.test( + redirectionURL, + ); // doesn't start with a protocol scheme and "//" (e.g., "http://", "https://", "//") + + let finalRedirectionURL; + + if (isRelativeRedirectionURL) { + finalRedirectionURL = redirectionURL; + } else { + let remappedRedirectionURL = this.resolveURLMapping( + redirectionURL, + 'virtual-to-real', + ); + finalRedirectionURL = remappedRedirectionURL || redirectionURL; + } + response.headers.set('Location', finalRedirectionURL); + } + } + + private async runFetch(request: Request, init?: RequestInit) { for (let handler of this.handlers) { let response = await handler(request); if (response) { @@ -75,7 +189,12 @@ export class VirtualNetwork { } return this.nativeFetch(request, init); - }; + } + + createEventSource(url: string) { + let mappedUrl = this.resolveURLMapping(url, 'virtual-to-real'); + return new EventSource(mappedUrl || url); + } } function isUrlLike(moduleIdentifier: string): boolean { @@ -86,3 +205,63 @@ function isUrlLike(moduleIdentifier: string): boolean { moduleIdentifier.startsWith('https://') ); } + +async function getContentOfReadableStream( + requestBody: ReadableStream | null, +): Promise { + if (requestBody) { + let isPending = true; + let arrayLength = 0; + let unit8Arrays = []; + let reader = requestBody.getReader(); + do { + let readableResults = await reader.read(); + + if (readableResults.value) { + arrayLength += readableResults.value.length; + unit8Arrays.push(readableResults.value); + } + + isPending = !readableResults.done; + } while (isPending); + let mergedArray = new Uint8Array(arrayLength); + unit8Arrays.forEach((array) => mergedArray.set(array)); + return mergedArray; + } + return null; +} + +async function buildRequest(url: string, originalRequest: Request) { + if (url === originalRequest.url) { + return originalRequest; + } + + // To reach the goal of creating a new Request but with a different url it is + // usually enough to create a new Request object with the new url and the same + // properties as the original request, but there are issues when the body is + // a ReadableStream - browser reports the following error: + // "TypeError: Failed to construct 'Request': The `duplex` member must be + // specified for a request with a streaming body." Even adding the `duplex` + // property will not fix the issue - the browser request being made to + // our local server then expects HTTP/2 connection which is currently not + // supported in our local server. To avoid all these issues, we resort to + // reading the body of the original request and creating a new Request with + // the new url and the body as a Uint8Array. + + let body = null; + if (originalRequest.body) { + body = await getContentOfReadableStream(originalRequest.clone().body); + } + return new Request(url, { + method: originalRequest.method, + headers: originalRequest.headers, + body, + referrer: originalRequest.referrer, + referrerPolicy: originalRequest.referrerPolicy, + mode: originalRequest.mode, + credentials: originalRequest.credentials, + cache: originalRequest.cache, + redirect: originalRequest.redirect, + integrity: originalRequest.integrity, + }); +}