Skip to content

Commit

Permalink
feat(gatsby-plugin-sharp): add experimental opt-in lazy image process…
Browse files Browse the repository at this point in the history 
…ing mode for `gatsby develop` (#28288)

* feat(gatsby-plugin-sharp): enable lazy image processing

* fix progress bar and cloud

* track usage of query on demand feature

* Use GATSBY_EXPERIMENTAL_LAZY_IMAGES vs `experimentalDisableLazyProcessing` plugin option

* Don't hook to express when disabled

* Update test snapshot

* fix: restore behavior from master for onCreateDevServer

* Add a note on FAST_DEV compatibility with gatsby-plugin-sharp

* Reset GATSBY_EXPERIMENTAL_LAZY_IMAGES for non-develop and CI

Co-authored-by: Ward Peeters <[email protected]>
Co-authored-by: Michal Piechowiak <[email protected]>
  • Loading branch information
@vladar @wardpeet @pieh
3 people authored Nov 26, 2020
1 parent 3666130 commit cc68a1f
Show file tree
Hide file tree
Showing 6 changed files with 174 additions and 5 deletions.
1 change: 1 addition & 0 deletions packages/gatsby-plugin-sharp/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"bluebird": "^3.7.2",
"fs-extra": "^9.0.1",
"gatsby-core-utils": "^1.6.0-next.0",
"gatsby-telemetry": "^1.6.0-next.1",
"got": "^10.7.0",
"imagemin": "^7.0.1",
"imagemin-mozjpeg": "^9.0.0",
Expand Down
25 changes: 25 additions & 0 deletions packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ exports[`gatsby-plugin-sharp fixed correctly infers the width when only the heig
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -114,6 +115,7 @@ exports[`gatsby-plugin-sharp fixed does not warn when the requested width is equ
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -170,6 +172,7 @@ exports[`gatsby-plugin-sharp fixed should give the same result with createJob as
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -226,6 +229,7 @@ exports[`gatsby-plugin-sharp fixed warns when the requested width is greater tha
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -268,6 +272,7 @@ exports[`gatsby-plugin-sharp fluid accepts srcSet breakpoints 1`] = `
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -338,6 +343,7 @@ exports[`gatsby-plugin-sharp fluid adds pathPrefix if defined 1`] = `
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -379,6 +385,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -454,6 +461,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -535,6 +543,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -616,6 +625,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -697,6 +707,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -756,6 +767,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -819,6 +831,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -882,6 +895,7 @@ Array [
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -946,6 +960,7 @@ exports[`gatsby-plugin-sharp fluid does not change the arguments object it is gi
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -988,6 +1003,7 @@ exports[`gatsby-plugin-sharp fluid ensure maxWidth is in srcSet breakpoints 1`]
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1092,6 +1108,7 @@ exports[`gatsby-plugin-sharp fluid infers the maxWidth if only maxHeight is give
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1169,6 +1186,7 @@ exports[`gatsby-plugin-sharp fluid keeps original file name 1`] = `
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1211,6 +1229,7 @@ exports[`gatsby-plugin-sharp fluid prevents duplicate breakpoints 1`] = `
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1274,6 +1293,7 @@ exports[`gatsby-plugin-sharp fluid reject any breakpoints larger than the origin
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1344,6 +1364,7 @@ exports[`gatsby-plugin-sharp fluid should give the same result with createJob as
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1401,6 +1422,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJob file name works w
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1445,6 +1467,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJob should round heig
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1489,6 +1512,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJobV2 file name works
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down Expand Up @@ -1531,6 +1555,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJobV2 should round he
Array [
Object {
"args": Object {
"isLazy": false,
"operations": Array [
Object {
"args": Object {
Expand Down
125 changes: 123 additions & 2 deletions packages/gatsby-plugin-sharp/src/gatsby-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,45 @@ const {
setBoundActionCreators,
// queue: jobQueue,
// reportError,
_unstable_createJob,
} = require(`./index`)
const { pathExists } = require(`fs-extra`)
const { slash, isCI } = require(`gatsby-core-utils`)
const { trackFeatureIsUsed } = require(`gatsby-telemetry`)
const { getProgressBar, createOrGetProgressBar } = require(`./utils`)

const { setPluginOptions } = require(`./plugin-options`)
const path = require(`path`)

function prepareLazyImagesExperiment(reporter) {
if (!process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES) {
return
}
if (process.env.gatsby_executing_command !== `develop`) {
// We don't want to ever have this flag enabled for anything other than develop
// in case someone have this env var globally set
delete process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES
return
}
if (isCI()) {
delete process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES
reporter.warn(
`Lazy Image Processing experiment is not available in CI environment. Continuing with regular mode.`
)
return
}
// We show a different notice for GATSBY_EXPERIMENTAL_FAST_DEV umbrella
if (!process.env.GATSBY_EXPERIMENTAL_FAST_DEV) {
reporter.info(
`[gatsby-plugin-sharp] The lazy image processing experiment is enabled`
)
}
trackFeatureIsUsed(`LazyImageProcessing`)
}

exports.onPreInit = ({ reporter }) => {
prepareLazyImagesExperiment(reporter)
}

// create the progressbar once and it will be killed in another lifecycle
const finishProgressBar = () => {
Expand All @@ -16,9 +51,62 @@ const finishProgressBar = () => {
}

exports.onPostBuild = () => finishProgressBar()
exports.onCreateDevServer = () => finishProgressBar()

exports.onPreBootstrap = ({ actions, emitter, reporter }, pluginOptions) => {
exports.onCreateDevServer = async ({ app, cache, reporter }) => {
if (!process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES) {
finishProgressBar()
return
}

createOrGetProgressBar()
finishProgressBar()

app.use(async (req, res, next) => {
const pathOnDisk = path.resolve(path.join(`./public/`, req.url))

if (await pathExists(pathOnDisk)) {
return res.sendFile(pathOnDisk)
}

const jobContentDigest = await cache.get(req.url)
const cacheResult = jobContentDigest
? await cache.get(jobContentDigest)
: null

if (!cacheResult) {
return next()
}

await _unstable_createJob(cacheResult, { reporter })
// we should implement cache.del inside our abstraction
await cache.cache.del(jobContentDigest)
await cache.cache.del(req.url)

return res.sendFile(pathOnDisk)
})
}

// So something is wrong with the reporter, when I do this in preBootstrap,
// the progressbar gets not updated
exports.onPostBootstrap = async ({ reporter, cache, store }) => {
if (process.env.gatsby_executing_command !== `develop`) {
// recreate jobs that haven't been triggered by develop yet
// removing stale jobs has already kicked in so we know these still need to process
for (const [contentDigest] of store.getState().jobsV2.complete) {
const job = await cache.get(contentDigest)

if (job) {
// we dont have to await, gatsby does this for us
_unstable_createJob(job, { reporter })
}
}
}
}

exports.onPreBootstrap = async (
{ actions, emitter, reporter, cache, store },
pluginOptions
) => {
setBoundActionCreators(actions)
setPluginOptions(pluginOptions)

Expand All @@ -40,6 +128,33 @@ exports.onPreBootstrap = ({ actions, emitter, reporter }, pluginOptions) => {

emitter.on(`CREATE_JOB_V2`, action => {
if (action.plugin.name === `gatsby-plugin-sharp`) {
if (action.payload.job.args.isLazy) {
// we have to remove some internal pieces
const job = {
name: action.payload.job.name,
inputPaths: action.payload.job.inputPaths.map(input => input.path),
outputDir: action.payload.job.outputDir,
args: {
...action.payload.job.args,
isLazy: false,
},
}
cache.set(action.payload.job.contentDigest, job)

action.payload.job.args.operations.forEach(op => {
const cacheKey = slash(
path.relative(
path.join(process.cwd(), `public`),
path.join(action.payload.job.outputDir, op.outputPath)
)
)

cache.set(`/${cacheKey}`, action.payload.job.contentDigest)
})

return
}

const job = action.payload.job
const imageCount = job.args.operations.length
imageCountInJobsMap.set(job.contentDigest, imageCount)
Expand All @@ -51,6 +166,12 @@ exports.onPreBootstrap = ({ actions, emitter, reporter }, pluginOptions) => {
emitter.on(`END_JOB_V2`, action => {
if (action.plugin.name === `gatsby-plugin-sharp`) {
const jobContentDigest = action.payload.jobContentDigest

// when it's lazy we didn't set it
if (!imageCountInJobsMap.has(jobContentDigest)) {
return
}

const imageCount = imageCountInJobsMap.get(jobContentDigest)
const progress = createOrGetProgressBar(reporter)
progress.tick(imageCount)
Expand Down
9 changes: 7 additions & 2 deletions packages/gatsby-plugin-sharp/src/gatsby-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,12 @@ const q = queue(
* @param {{inputPaths: string[], outputDir: string, args: WorkerInput}} args
* @return Promise
*/
exports.IMAGE_PROCESSING = ({ inputPaths, outputDir, args }) =>
new Promise((resolve, reject) => {
exports.IMAGE_PROCESSING = ({ inputPaths, outputDir, args }) => {
if (args.isLazy) {
return Promise.resolve()
}

return new Promise((resolve, reject) => {
q.push({ inputPaths, outputDir, args }, function (err) {
if (err) {
return reject(err)
Expand All @@ -48,3 +52,4 @@ exports.IMAGE_PROCESSING = ({ inputPaths, outputDir, args }) =>
return resolve()
})
})
}
Loading

0 comments on commit cc68a1f

Please sign in to comment.