Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix image integration crash on Netlify Functions due to import.meta.url #5888

Merged
merged 2 commits into from
Jan 19, 2023
Merged

Fix image integration crash on Netlify Functions due to import.meta.url #5888

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ec820af
fix(image): Fix immediate crash on Netlify functions due to `import.m…
Princesseuh Jan 18, 2023
9e6be6d
chore: changeset
Princesseuh Jan 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/twenty-boxes-know.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@astrojs/image': patch
---

Fix crash on Netlify Functions due to `import.meta.url`
6 changes: 3 additions & 3 deletions packages/integrations/image/src/vendor/squoosh/avif/avif_node_dec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/* eslint-disable */
// @ts-nocheck
import { createRequire } from 'module';
import { dirname } from '../emscripten-utils.js';
const require = createRequire(import.meta.url);
import { dirname, getModuleURL } from '../emscripten-utils.js';
const require = createRequire(getModuleURL(import.meta.url));

var Module = (function () {
return function (Module) {
Expand Down Expand Up @@ -43,7 +43,7 @@ var Module = (function () {
if (ENVIRONMENT_IS_WORKER) {
scriptDirectory = require('path').dirname(scriptDirectory) + '/'
} else {
scriptDirectory = dirname(import.meta.url) + '/'
scriptDirectory = dirname(getModuleURL(import.meta.url)) + '/'
}
read_ = function shell_read(filename, binary) {
if (!nodeFS) nodeFS = require('fs')
Expand Down
6 changes: 3 additions & 3 deletions packages/integrations/image/src/vendor/squoosh/avif/avif_node_enc.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/* eslint-disable */
// @ts-nocheck
import { createRequire } from 'module';
import { dirname } from '../emscripten-utils.js';
const require = createRequire(import.meta.url);
import { dirname, getModuleURL } from '../emscripten-utils.js';
const require = createRequire(getModuleURL(import.meta.url));

var Module = (function () {
return function (Module) {
Expand Down Expand Up @@ -43,7 +43,7 @@ var Module = (function () {
if (ENVIRONMENT_IS_WORKER) {
scriptDirectory = require('path').dirname(scriptDirectory) + '/'
} else {
scriptDirectory = dirname(import.meta.url) + '/'
scriptDirectory = dirname(getModuleURL(import.meta.url)) + '/'
}
read_ = function shell_read(filename, binary) {
if (!nodeFS) nodeFS = require('fs')
Expand Down
22 changes: 11 additions & 11 deletions packages/integrations/image/src/vendor/squoosh/codecs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { promises as fsp } from 'node:fs'
import { instantiateEmscriptenWasm, pathify } from './emscripten-utils.js'
import { getModuleURL, instantiateEmscriptenWasm, pathify } from './emscripten-utils.js'

interface DecodeModule extends EmscriptenWasm.Module {
decode: (data: Uint8Array) => ImageData
Expand Down Expand Up @@ -37,50 +37,50 @@ export interface RotateOptions {
import type { MozJPEGModule as MozJPEGEncodeModule } from './mozjpeg/mozjpeg_enc'
// @ts-ignore
import mozEnc from './mozjpeg/mozjpeg_node_enc.js'
const mozEncWasm = new URL('./mozjpeg/mozjpeg_node_enc.wasm', import.meta.url)
const mozEncWasm = new URL('./mozjpeg/mozjpeg_node_enc.wasm', getModuleURL(import.meta.url))
// @ts-ignore
import mozDec from './mozjpeg/mozjpeg_node_dec.js'
const mozDecWasm = new URL('./mozjpeg/mozjpeg_node_dec.wasm', import.meta.url)
const mozDecWasm = new URL('./mozjpeg/mozjpeg_node_dec.wasm', getModuleURL(import.meta.url))

// WebP
import type { WebPModule as WebPEncodeModule } from './webp/webp_enc'
// @ts-ignore
import webpEnc from './webp/webp_node_enc.js'
const webpEncWasm = new URL('./webp/webp_node_enc.wasm', import.meta.url)
const webpEncWasm = new URL('./webp/webp_node_enc.wasm', getModuleURL(import.meta.url))
// @ts-ignore
import webpDec from './webp/webp_node_dec.js'
const webpDecWasm = new URL('./webp/webp_node_dec.wasm', import.meta.url)
const webpDecWasm = new URL('./webp/webp_node_dec.wasm', getModuleURL(import.meta.url))

// AVIF
import type { AVIFModule as AVIFEncodeModule } from './avif/avif_enc'
// @ts-ignore
import avifEnc from './avif/avif_node_enc.js'
const avifEncWasm = new URL('./avif/avif_node_enc.wasm', import.meta.url)
const avifEncWasm = new URL('./avif/avif_node_enc.wasm', getModuleURL(import.meta.url))
// @ts-ignore
import avifDec from './avif/avif_node_dec.js'
const avifDecWasm = new URL('./avif/avif_node_dec.wasm', import.meta.url)
const avifDecWasm = new URL('./avif/avif_node_dec.wasm', getModuleURL(import.meta.url))

// PNG
// @ts-ignore
import * as pngEncDec from './png/squoosh_png.js'
const pngEncDecWasm = new URL('./png/squoosh_png_bg.wasm', import.meta.url)
const pngEncDecWasm = new URL('./png/squoosh_png_bg.wasm', getModuleURL(import.meta.url))
const pngEncDecInit = () =>
pngEncDec.default(fsp.readFile(pathify(pngEncDecWasm.toString())))

// OxiPNG
// @ts-ignore
import * as oxipng from './png/squoosh_oxipng.js'
const oxipngWasm = new URL('./png/squoosh_oxipng_bg.wasm', import.meta.url)
const oxipngWasm = new URL('./png/squoosh_oxipng_bg.wasm', getModuleURL(import.meta.url))
const oxipngInit = () => oxipng.default(fsp.readFile(pathify(oxipngWasm.toString())))

// Resize
// @ts-ignore
import * as resize from './resize/squoosh_resize.js'
const resizeWasm = new URL('./resize/squoosh_resize_bg.wasm', import.meta.url)
const resizeWasm = new URL('./resize/squoosh_resize_bg.wasm', getModuleURL(import.meta.url))
const resizeInit = () => resize.default(fsp.readFile(pathify(resizeWasm.toString())))

// rotate
const rotateWasm = new URL('./rotate/rotate.wasm', import.meta.url)
const rotateWasm = new URL('./rotate/rotate.wasm', getModuleURL(import.meta.url))

// Our decoders currently rely on a `ImageData` global.
import ImageData from './image_data.js'
Expand Down
17 changes: 15 additions & 2 deletions packages/integrations/image/src/vendor/squoosh/emscripten-utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//
import { fileURLToPath } from 'node:url'
//
import { fileURLToPath, pathToFileURL } from 'node:url'

export function pathify(path: string): string {
if (path.startsWith('file://')) {
Expand Down Expand Up @@ -29,3 +29,16 @@ export function instantiateEmscriptenWasm<T extends EmscriptenWasm.Module>(
export function dirname(url: string) {
return url.substring(0, url.lastIndexOf('/'))
}

/**
* On certain serverless hosts, our ESM bundle is transpiled to CJS before being run, which means
* import.meta.url is undefined, so we'll fall back to __dirname in those cases
* We should be able to remove this once https://github.com/netlify/zip-it-and-ship-it/issues/750 is fixed
*/
export function getModuleURL(url: string | undefined): string {
if (!url) {
return pathToFileURL(__dirname).toString();
Copy link
Member

@bluwy bluwy Jan 18, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The __dirname refers relative to this utils file. Would it work correctly if it's used by the consumer file later on?

I see some doing

scriptDirectory = dirname(getModuleURL(import.meta.url)) + '/'

which might have a different value depending on whether import.meta.url is supported.

Copy link
Member Author

@Princesseuh Princesseuh Jan 19, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's admittedly not ideal, but it does work because the __dirname call is only used in SSR where everything gets bundled 😅

If you have a better solution, I'll definitely take! Got lost in between the __dirname, __filename and import.meta.url while working on this so really I settled on the solution that actually did work, but I'm not picky 😵‍💫

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I see. Didn't know this was bundled so it would always have the same value.

This trick sounds good to me then, though I wonder if it should be __filename instead since that reflects import.meta.url. If you've tested with __dirname and it works then I'll be fine by it too 👍

}

return url
}
152 changes: 81 additions & 71 deletions packages/integrations/image/src/vendor/squoosh/image-pool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,137 +4,147 @@ import { fileURLToPath } from 'url';
import type { OutputFormat } from '../../loaders/index.js';
import execOnce from '../../utils/execOnce.js';
import WorkerPool from '../../utils/workerPool.js';
import { getModuleURL } from './emscripten-utils.js';
import type { Operation } from './image.js';
import * as impl from './impl.js';

const getWorker = execOnce(
() => {
return new WorkerPool(
// There will be at most 7 workers needed since each worker will take
// at least 1 operation type.
Math.max(1, Math.min(cpus().length - 1, 7)),
fileURLToPath(import.meta.url)
);
}
)
const getWorker = execOnce(() => {
return new WorkerPool(
// There will be at most 7 workers needed since each worker will take
// at least 1 operation type.
Math.max(1, Math.min(cpus().length - 1, 7)),
fileURLToPath(getModuleURL(import.meta.url))
);
});

type DecodeParams = {
operation: 'decode',
buffer: Buffer
operation: 'decode';
buffer: Buffer;
};
type ResizeParams = {
operation: 'resize',
imageData: ImageData,
height?: number,
width?: number
operation: 'resize';
imageData: ImageData;
height?: number;
width?: number;
};
type RotateParams = {
operation: 'rotate',
imageData: ImageData,
numRotations: number
operation: 'rotate';
imageData: ImageData;
numRotations: number;
};
type EncodeAvifParams = {
operation: 'encodeavif',
imageData: ImageData,
quality: number
}
operation: 'encodeavif';
imageData: ImageData;
quality: number;
};
type EncodeJpegParams = {
operation: 'encodejpeg',
imageData: ImageData,
quality: number
}
operation: 'encodejpeg';
imageData: ImageData;
quality: number;
};
type EncodePngParams = {
operation: 'encodepng',
imageData: ImageData
}
operation: 'encodepng';
imageData: ImageData;
};
type EncodeWebpParams = {
operation: 'encodewebp',
imageData: ImageData,
quality: number
}
type JobMessage = DecodeParams | ResizeParams | RotateParams | EncodeAvifParams | EncodeJpegParams | EncodePngParams | EncodeWebpParams
operation: 'encodewebp';
imageData: ImageData;
quality: number;
};
type JobMessage =
| DecodeParams
| ResizeParams
| RotateParams
| EncodeAvifParams
| EncodeJpegParams
| EncodePngParams
| EncodeWebpParams;

function handleJob(params: JobMessage) {
switch (params.operation) {
case 'decode':
return impl.decodeBuffer(params.buffer)
switch (params.operation) {
case 'decode':
return impl.decodeBuffer(params.buffer);
case 'resize':
return impl.resize({ image: params.imageData as any, width: params.width, height: params.height })
return impl.resize({
image: params.imageData as any,
width: params.width,
height: params.height,
});
case 'rotate':
return impl.rotate(params.imageData as any, params.numRotations);
case 'encodeavif':
return impl.encodeAvif(params.imageData as any, { quality: params.quality })
return impl.encodeAvif(params.imageData as any, { quality: params.quality });
case 'encodejpeg':
return impl.encodeJpeg(params.imageData as any, { quality: params.quality })
return impl.encodeJpeg(params.imageData as any, { quality: params.quality });
case 'encodepng':
return impl.encodePng(params.imageData as any)
return impl.encodePng(params.imageData as any);
case 'encodewebp':
return impl.encodeWebp(params.imageData as any, { quality: params.quality })
default:
throw Error(`Invalid job "${(params as any).operation}"`);
}
return impl.encodeWebp(params.imageData as any, { quality: params.quality });
default:
throw Error(`Invalid job "${(params as any).operation}"`);
}
}

export async function processBuffer(
buffer: Buffer,
operations: Operation[],
encoding: OutputFormat,
quality?: number
buffer: Buffer,
operations: Operation[],
encoding: OutputFormat,
quality?: number
): Promise<Uint8Array> {
// @ts-ignore
const worker = await getWorker()
const worker = await getWorker();

let imageData = await worker.dispatchJob({
let imageData = await worker.dispatchJob({
operation: 'decode',
buffer,
})
for (const operation of operations) {
if (operation.type === 'rotate') {
});
for (const operation of operations) {
if (operation.type === 'rotate') {
imageData = await worker.dispatchJob({
operation: 'rotate',
imageData,
numRotations: operation.numRotations
numRotations: operation.numRotations,
});
} else if (operation.type === 'resize') {
} else if (operation.type === 'resize') {
imageData = await worker.dispatchJob({
operation: 'resize',
imageData,
height: operation.height,
width: operation.width
})
}
}
width: operation.width,
});
}
}

switch (encoding) {
case 'avif':
return await worker.dispatchJob({
return (await worker.dispatchJob({
operation: 'encodeavif',
imageData,
quality,
}) as Uint8Array;
})) as Uint8Array;
case 'jpeg':
case 'jpg':
return await worker.dispatchJob({
return (await worker.dispatchJob({
operation: 'encodejpeg',
imageData,
quality,
}) as Uint8Array;
})) as Uint8Array;
case 'png':
return await worker.dispatchJob({
return (await worker.dispatchJob({
operation: 'encodepng',
imageData,
}) as Uint8Array;
})) as Uint8Array;
case 'webp':
return await worker.dispatchJob({
return (await worker.dispatchJob({
operation: 'encodewebp',
imageData,
quality,
}) as Uint8Array;
})) as Uint8Array;
default:
throw Error(`Unsupported encoding format`)
throw Error(`Unsupported encoding format`);
}
}

if (!isMainThread) {
WorkerPool.useThisThreadAsWorker(handleJob);
WorkerPool.useThisThreadAsWorker(handleJob);
}
6 changes: 3 additions & 3 deletions packages/integrations/image/src/vendor/squoosh/mozjpeg/mozjpeg_node_dec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/* eslint-disable */
// @ts-nocheck
import { createRequire } from 'module';
import { dirname } from '../emscripten-utils.js';
const require = createRequire(import.meta.url);
import { dirname, getModuleURL } from '../emscripten-utils.js';
const require = createRequire(getModuleURL(import.meta.url));

var Module = (function () {
return function (Module) {
Expand Down Expand Up @@ -43,7 +43,7 @@ var Module = (function () {
if (ENVIRONMENT_IS_WORKER) {
scriptDirectory = require('path').dirname(scriptDirectory) + '/'
} else {
scriptDirectory = dirname(import.meta.url) + '/'
scriptDirectory = dirname(getModuleURL(import.meta.url)) + '/'
}
read_ = function shell_read(filename, binary) {
if (!nodeFS) nodeFS = require('fs')
Expand Down
6 changes: 3 additions & 3 deletions packages/integrations/image/src/vendor/squoosh/mozjpeg/mozjpeg_node_enc.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/* eslint-disable */
// @ts-nocheck
import { createRequire } from 'module';
import { dirname } from '../emscripten-utils.js';
const require = createRequire(import.meta.url);
import { dirname, getModuleURL } from '../emscripten-utils.js';
const require = createRequire(getModuleURL(import.meta.url));

var Module = (function () {
return function (Module) {
Expand Down Expand Up @@ -43,7 +43,7 @@ var Module = (function () {
if (ENVIRONMENT_IS_WORKER) {
scriptDirectory = require('path').dirname(scriptDirectory) + '/'
} else {
scriptDirectory = dirname(import.meta.url) + '/'
scriptDirectory = dirname(getModuleURL(import.meta.url)) + '/'
}
read_ = function shell_read(filename, binary) {
if (!nodeFS) nodeFS = require('fs')
Expand Down
Loading