From 7d410cab09e7bdcec30673494eb3817d0a774131 Mon Sep 17 00:00:00 2001 From: Lukasz Ostrowski Date: Fri, 22 Sep 2023 09:05:24 +0200 Subject: [PATCH 1/3] Product feed - change max function exec time to 5 minutes --- .changeset/tender-seas-hunt.md | 5 +++++ .../src/pages/api/feed/[url]/[channel]/google.xml.ts | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 .changeset/tender-seas-hunt.md diff --git a/.changeset/tender-seas-hunt.md b/.changeset/tender-seas-hunt.md new file mode 100644 index 000000000..e76b5d29f --- /dev/null +++ b/.changeset/tender-seas-hunt.md @@ -0,0 +1,5 @@ +--- +"saleor-app-products-feed": patch +--- + +Changed Vercel's maximum execution time to be 5 minutes for feed generation. This should help with the previous limits of 60s, that was not enough for feed to be generated. diff --git a/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts b/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts index 7f9718895..20ea6f000 100644 --- a/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts +++ b/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts @@ -16,6 +16,10 @@ import { getDownloadUrl, getFileName } from "../../../../../modules/file-storage import { RootConfig } from "../../../../../modules/app-configuration/app-config"; import { z, ZodError } from "zod"; +export const config = { + maxDuration: 5, +}; + // By default we cache the feed for 5 minutes. This can be changed by setting the FEED_CACHE_MAX_AGE const FEED_CACHE_MAX_AGE = process.env.FEED_CACHE_MAX_AGE ? parseInt(process.env.FEED_CACHE_MAX_AGE, 10) From 99f0cd322398eb8231cf1a43cdc6dbaadc8795c0 Mon Sep 17 00:00:00 2001 From: Krzysztof Wolski Date: Fri, 22 Sep 2023 12:44:24 +0200 Subject: [PATCH 2/3] Remove cache update on product webhooks --- .changeset/breezy-carrots-reply.md | 5 +++ .../ProductVariantWebhookPayload.graphql | 8 ---- .../fragments/ProductWebhookPayload.graphql | 8 ---- .../subscriptions/productCreated.graphql | 9 ---- .../subscriptions/productDeleted.graphql | 9 ---- .../subscriptions/productUpdated.graphql | 9 ---- .../productVariantCreated.graphql | 9 ---- .../productVariantDeleted.graphql | 10 ----- .../productVariantUpdated.graphql | 9 ---- .../app-configuration.router.ts | 12 ------ .../metadata-cache/cache-configurator.ts | 33 --------------- .../update-cache-for-configurations.ts | 33 --------------- .../metadata-cache/update-cache-on-webhook.ts | 41 ------------------- .../api/feed/[url]/[channel]/google.xml.ts | 20 +-------- apps/products-feed/src/pages/api/manifest.ts | 13 +----- .../src/pages/api/webhooks/product_created.ts | 41 ------------------- .../src/pages/api/webhooks/product_deleted.ts | 40 ------------------ .../src/pages/api/webhooks/product_updated.ts | 40 ------------------ .../api/webhooks/product_variant_created.ts | 41 ------------------- .../api/webhooks/product_variant_deleted.ts | 41 ------------------- .../api/webhooks/product_variant_updated.ts | 41 ------------------- 21 files changed, 8 insertions(+), 464 deletions(-) create mode 100644 .changeset/breezy-carrots-reply.md delete mode 100644 apps/products-feed/graphql/fragments/ProductVariantWebhookPayload.graphql delete mode 100644 apps/products-feed/graphql/fragments/ProductWebhookPayload.graphql delete mode 100644 apps/products-feed/graphql/subscriptions/productCreated.graphql delete mode 100644 apps/products-feed/graphql/subscriptions/productDeleted.graphql delete mode 100644 apps/products-feed/graphql/subscriptions/productUpdated.graphql delete mode 100644 apps/products-feed/graphql/subscriptions/productVariantCreated.graphql delete mode 100644 apps/products-feed/graphql/subscriptions/productVariantDeleted.graphql delete mode 100644 apps/products-feed/graphql/subscriptions/productVariantUpdated.graphql delete mode 100644 apps/products-feed/src/modules/metadata-cache/cache-configurator.ts delete mode 100644 apps/products-feed/src/modules/metadata-cache/update-cache-for-configurations.ts delete mode 100644 apps/products-feed/src/modules/metadata-cache/update-cache-on-webhook.ts delete mode 100644 apps/products-feed/src/pages/api/webhooks/product_created.ts delete mode 100644 apps/products-feed/src/pages/api/webhooks/product_deleted.ts delete mode 100644 apps/products-feed/src/pages/api/webhooks/product_updated.ts delete mode 100644 apps/products-feed/src/pages/api/webhooks/product_variant_created.ts delete mode 100644 apps/products-feed/src/pages/api/webhooks/product_variant_deleted.ts delete mode 100644 apps/products-feed/src/pages/api/webhooks/product_variant_updated.ts diff --git a/.changeset/breezy-carrots-reply.md b/.changeset/breezy-carrots-reply.md new file mode 100644 index 000000000..4c78fcc1b --- /dev/null +++ b/.changeset/breezy-carrots-reply.md @@ -0,0 +1,5 @@ +--- +"saleor-app-products-feed": patch +--- + +Removed webhooks on product changes used for feed cache due to changed max execution time. diff --git a/apps/products-feed/graphql/fragments/ProductVariantWebhookPayload.graphql b/apps/products-feed/graphql/fragments/ProductVariantWebhookPayload.graphql deleted file mode 100644 index 7f3215012..000000000 --- a/apps/products-feed/graphql/fragments/ProductVariantWebhookPayload.graphql +++ /dev/null @@ -1,8 +0,0 @@ -fragment ProductVariantWebhookPayload on ProductVariant { - channel - channelListings { - channel { - slug - } - } -} diff --git a/apps/products-feed/graphql/fragments/ProductWebhookPayload.graphql b/apps/products-feed/graphql/fragments/ProductWebhookPayload.graphql deleted file mode 100644 index 7a49bac4f..000000000 --- a/apps/products-feed/graphql/fragments/ProductWebhookPayload.graphql +++ /dev/null @@ -1,8 +0,0 @@ -fragment ProductWebhookPayload on Product { - channel - channelListings { - channel { - slug - } - } -} diff --git a/apps/products-feed/graphql/subscriptions/productCreated.graphql b/apps/products-feed/graphql/subscriptions/productCreated.graphql deleted file mode 100644 index f55430ac3..000000000 --- a/apps/products-feed/graphql/subscriptions/productCreated.graphql +++ /dev/null @@ -1,9 +0,0 @@ -subscription ProductCreated { - event { - ... on ProductCreated { - product { - ...ProductWebhookPayload - } - } - } -} diff --git a/apps/products-feed/graphql/subscriptions/productDeleted.graphql b/apps/products-feed/graphql/subscriptions/productDeleted.graphql deleted file mode 100644 index dda49e0b2..000000000 --- a/apps/products-feed/graphql/subscriptions/productDeleted.graphql +++ /dev/null @@ -1,9 +0,0 @@ -subscription ProductDeleted { - event { - ... on ProductDeleted { - product { - ...ProductWebhookPayload - } - } - } -} diff --git a/apps/products-feed/graphql/subscriptions/productUpdated.graphql b/apps/products-feed/graphql/subscriptions/productUpdated.graphql deleted file mode 100644 index 9d76752a8..000000000 --- a/apps/products-feed/graphql/subscriptions/productUpdated.graphql +++ /dev/null @@ -1,9 +0,0 @@ -subscription ProductUpdated { - event { - ... on ProductUpdated { - product { - ...ProductWebhookPayload - } - } - } -} diff --git a/apps/products-feed/graphql/subscriptions/productVariantCreated.graphql b/apps/products-feed/graphql/subscriptions/productVariantCreated.graphql deleted file mode 100644 index 81fbf7ddb..000000000 --- a/apps/products-feed/graphql/subscriptions/productVariantCreated.graphql +++ /dev/null @@ -1,9 +0,0 @@ -subscription ProductVariantCreated { - event { - ... on ProductVariantCreated { - productVariant { - ...ProductVariantWebhookPayload - } - } - } -} diff --git a/apps/products-feed/graphql/subscriptions/productVariantDeleted.graphql b/apps/products-feed/graphql/subscriptions/productVariantDeleted.graphql deleted file mode 100644 index a313a6807..000000000 --- a/apps/products-feed/graphql/subscriptions/productVariantDeleted.graphql +++ /dev/null @@ -1,10 +0,0 @@ -subscription ProductVariantDeleted { - event { - ... on ProductVariantDeleted { - productVariant { - ...ProductVariantWebhookPayload - } - } - - } -} diff --git a/apps/products-feed/graphql/subscriptions/productVariantUpdated.graphql b/apps/products-feed/graphql/subscriptions/productVariantUpdated.graphql deleted file mode 100644 index 301ac9960..000000000 --- a/apps/products-feed/graphql/subscriptions/productVariantUpdated.graphql +++ /dev/null @@ -1,9 +0,0 @@ -subscription ProductVariantUpdated { - event { - ... on ProductVariantUpdated { - productVariant { - ...ProductVariantWebhookPayload - } - } - } -} diff --git a/apps/products-feed/src/modules/app-configuration/app-configuration.router.ts b/apps/products-feed/src/modules/app-configuration/app-configuration.router.ts index 4b2fd1327..d15f8c10a 100644 --- a/apps/products-feed/src/modules/app-configuration/app-configuration.router.ts +++ b/apps/products-feed/src/modules/app-configuration/app-configuration.router.ts @@ -2,7 +2,6 @@ import { router } from "../trpc/trpc-server"; import { protectedClientProcedure } from "../trpc/protected-client-procedure"; import { createLogger } from "@saleor/apps-shared"; -import { updateCacheForConfigurations } from "../metadata-cache/update-cache-for-configurations"; import { AppConfigSchema, imageSizeInputSchema, titleTemplateInputSchema } from "./app-config"; import { z } from "zod"; import { createS3ClientFromConfiguration } from "../file-storage/s3/create-s3-client-from-configuration"; @@ -106,17 +105,6 @@ export const appConfigurationRouter = router({ }) => { const config = await getConfig(); - /** - * TODO Check if this has to run, once its cached, it should be invalidated by webhooks only. - * - * But this operation isn't expensive and users will not continuously save this form - */ - await updateCacheForConfigurations({ - client: apiClient, - channelsSlugs: [input.channelSlug], - saleorApiUrl: saleorApiUrl, - }); - logger.debug({ channel: input.channelSlug }, "Updated cache for channel"); config.setChannelUrls(input.channelSlug, input.urls); diff --git a/apps/products-feed/src/modules/metadata-cache/cache-configurator.ts b/apps/products-feed/src/modules/metadata-cache/cache-configurator.ts deleted file mode 100644 index a272717ad..000000000 --- a/apps/products-feed/src/modules/metadata-cache/cache-configurator.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { SettingsManager } from "@saleor/app-sdk/settings-manager"; - -export class CacheConfigurator { - private metadataKeyPrefix = "cursor-cache-"; - - constructor(private metadataManager: SettingsManager, private saleorApiUrl: string) {} - - private constructKey(channel: string) { - return this.metadataKeyPrefix + channel; - } - - get({ channel }: { channel: string }): Promise { - return this.metadataManager.get(this.constructKey(channel), this.saleorApiUrl).then((data) => { - if (!data) { - return undefined; - } - - try { - return JSON.parse(data); - } catch (e) { - throw new Error("Invalid metadata value, can't be parsed"); - } - }); - } - - set({ channel, value }: { channel: string; value: string[] }): Promise { - return this.metadataManager.set({ - key: this.constructKey(channel), - value: JSON.stringify(value), - domain: this.saleorApiUrl, - }); - } -} diff --git a/apps/products-feed/src/modules/metadata-cache/update-cache-for-configurations.ts b/apps/products-feed/src/modules/metadata-cache/update-cache-for-configurations.ts deleted file mode 100644 index 36c81fed3..000000000 --- a/apps/products-feed/src/modules/metadata-cache/update-cache-for-configurations.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { createLogger } from "@saleor/apps-shared"; -import { CacheConfigurator } from "./cache-configurator"; -import { createSettingsManager } from "../../lib/metadata-manager"; -import { getCursors } from "../google-feed/fetch-product-data"; -import { Client } from "urql"; - -interface UpdateCacheForConfigurationsArgs { - client: Client; - saleorApiUrl: string; - channelsSlugs: string[]; -} - -export const updateCacheForConfigurations = async ({ - client, - channelsSlugs, - saleorApiUrl, -}: UpdateCacheForConfigurationsArgs) => { - const logger = createLogger({ saleorApiUrl: saleorApiUrl }); - - logger.debug("Updating the cursor cache"); - - const cache = new CacheConfigurator(createSettingsManager(client), saleorApiUrl); - - const cacheUpdatePromises = channelsSlugs.map(async (channel) => { - const cursors = await getCursors({ client, channel }); - - await cache.set({ channel, value: cursors }); - }); - - await Promise.all(cacheUpdatePromises); - - logger.debug("Cursor cache updated"); -}; diff --git a/apps/products-feed/src/modules/metadata-cache/update-cache-on-webhook.ts b/apps/products-feed/src/modules/metadata-cache/update-cache-on-webhook.ts deleted file mode 100644 index 06d12f564..000000000 --- a/apps/products-feed/src/modules/metadata-cache/update-cache-on-webhook.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { GraphqlClientFactory } from "../../lib/create-graphql-client"; -import { updateCacheForConfigurations } from "./update-cache-for-configurations"; -import { AuthData } from "@saleor/app-sdk/APL"; -import { - ProductVariantWebhookPayloadFragment, - ProductWebhookPayloadFragment, -} from "../../../generated/graphql"; -import { NextApiResponse } from "next"; - -type ChannelFragment = - | Pick - | Pick; - -export const updateCacheOnWebhook = async ({ - channels, - authData, - res, -}: { - authData: AuthData; - channels: ChannelFragment; - res: NextApiResponse; -}) => { - const client = GraphqlClientFactory.fromAuthData(authData); - - const channelsSlugs = [ - channels.channel, - ...(channels.channelListings?.map((cl) => cl.channel.slug) ?? []), - ].filter((c) => c) as string[]; - - if (channelsSlugs.length === 0) { - return res.status(200).end(); - } - - await updateCacheForConfigurations({ - channelsSlugs, - client, - saleorApiUrl: authData.saleorApiUrl, - }); - - return res.status(200).end(); -}; diff --git a/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts b/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts index 20ea6f000..275edf81c 100644 --- a/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts +++ b/apps/products-feed/src/pages/api/feed/[url]/[channel]/google.xml.ts @@ -6,9 +6,6 @@ import { fetchProductData } from "../../../../../modules/google-feed/fetch-produ import { GoogleFeedSettingsFetcher } from "../../../../../modules/google-feed/get-google-feed-settings"; import { generateGoogleXmlFeed } from "../../../../../modules/google-feed/generate-google-xml-feed"; import { fetchShopData } from "../../../../../modules/google-feed/fetch-shop-data"; -import { CacheConfigurator } from "../../../../../modules/metadata-cache/cache-configurator"; -import { createSettingsManager } from "../../../../../lib/metadata-manager"; -import { GraphqlClientFactory } from "../../../../../lib/create-graphql-client"; import { uploadFile } from "../../../../../modules/file-storage/s3/upload-file"; import { createS3ClientFromConfiguration } from "../../../../../modules/file-storage/s3/create-s3-client-from-configuration"; import { getFileDetails } from "../../../../../modules/file-storage/s3/get-file-details"; @@ -17,7 +14,7 @@ import { RootConfig } from "../../../../../modules/app-configuration/app-config" import { z, ZodError } from "zod"; export const config = { - maxDuration: 5, + maxDuration: 5 * 60, // 5 minutes }; // By default we cache the feed for 5 minutes. This can be changed by setting the FEED_CACHE_MAX_AGE @@ -161,23 +158,10 @@ export const handler = async (req: NextApiRequest, res: NextApiResponse) => { logger.debug("Generating a new feed"); - const cacheClient = GraphqlClientFactory.fromAuthData(authData); - - if (!cacheClient) { - logger.error("Can't create the gql client"); - return res.status(500).end(); - } - - // get cached cursors - const cache = new CacheConfigurator(createSettingsManager(cacheClient), authData.saleorApiUrl); - - const cursors = await cache.get({ channel }); - - // TODO: instead of separate variants, use group id https://support.google.com/merchants/answer/6324507?hl=en let productVariants: GoogleFeedProductVariantFragment[] = []; try { - productVariants = await fetchProductData({ client, channel, cursors, imageSize }); + productVariants = await fetchProductData({ client, channel, imageSize }); } catch (error) { logger.error(error); return res.status(400).end(); diff --git a/apps/products-feed/src/pages/api/manifest.ts b/apps/products-feed/src/pages/api/manifest.ts index f6db47bdc..b48d69d9b 100644 --- a/apps/products-feed/src/pages/api/manifest.ts +++ b/apps/products-feed/src/pages/api/manifest.ts @@ -2,11 +2,6 @@ import { createManifestHandler } from "@saleor/app-sdk/handlers/next"; import { AppManifest } from "@saleor/app-sdk/types"; import packageJson from "../../../package.json"; -import { webhookProductCreated } from "./webhooks/product_created"; -import { webhookProductDeleted } from "./webhooks/product_deleted"; -import { webhookProductVariantCreated } from "./webhooks/product_variant_created"; -import { webhookProductVariantDeleted } from "./webhooks/product_variant_deleted"; -import { webhookProductVariantUpdated } from "./webhooks/product_variant_updated"; export default createManifestHandler({ async manifestFactory({ appBaseUrl }) { @@ -31,13 +26,7 @@ export default createManifestHandler({ supportUrl: "https://github.com/saleor/apps/discussions", tokenTargetUrl: `${apiBaseURL}/api/register`, version: packageJson.version, - webhooks: [ - webhookProductCreated.getWebhookManifest(apiBaseURL), - webhookProductDeleted.getWebhookManifest(apiBaseURL), - webhookProductVariantCreated.getWebhookManifest(apiBaseURL), - webhookProductVariantDeleted.getWebhookManifest(apiBaseURL), - webhookProductVariantUpdated.getWebhookManifest(apiBaseURL), - ], + webhooks: [], }; return manifest; diff --git a/apps/products-feed/src/pages/api/webhooks/product_created.ts b/apps/products-feed/src/pages/api/webhooks/product_created.ts deleted file mode 100644 index d06aa92a9..000000000 --- a/apps/products-feed/src/pages/api/webhooks/product_created.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { NextWebhookApiHandler, SaleorAsyncWebhook } from "@saleor/app-sdk/handlers/next"; -import { createLogger } from "@saleor/apps-shared"; -import { saleorApp } from "../../../saleor-app"; -import { - ProductCreatedDocument, - ProductWebhookPayloadFragment, -} from "../../../../generated/graphql"; -import { updateCacheOnWebhook } from "../../../modules/metadata-cache/update-cache-on-webhook"; - -export const config = { - api: { - bodyParser: false, - }, -}; - -export const webhookProductCreated = new SaleorAsyncWebhook({ - webhookPath: "api/webhooks/product_created", - event: "PRODUCT_CREATED", - apl: saleorApp.apl, - query: ProductCreatedDocument, - // todo make it disabled by default, enable when app is configured - isActive: true, -}); - -const logger = createLogger({ - service: "webhook-product_created", -}); - -export const handler: NextWebhookApiHandler = async ( - req, - res, - context -) => { - await updateCacheOnWebhook({ - authData: context.authData, - channels: context.payload, - res, - }); -}; - -export default webhookProductCreated.createHandler(handler); diff --git a/apps/products-feed/src/pages/api/webhooks/product_deleted.ts b/apps/products-feed/src/pages/api/webhooks/product_deleted.ts deleted file mode 100644 index 98c94dcf4..000000000 --- a/apps/products-feed/src/pages/api/webhooks/product_deleted.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { NextWebhookApiHandler, SaleorAsyncWebhook } from "@saleor/app-sdk/handlers/next"; -import { - ProductDeletedDocument, - ProductWebhookPayloadFragment, -} from "../../../../generated/graphql"; -import { saleorApp } from "../../../saleor-app"; -import { createLogger } from "@saleor/apps-shared"; -import { updateCacheOnWebhook } from "../../../modules/metadata-cache/update-cache-on-webhook"; - -export const config = { - api: { - bodyParser: false, - }, -}; - -export const webhookProductDeleted = new SaleorAsyncWebhook({ - webhookPath: "api/webhooks/product_deleted", - event: "PRODUCT_DELETED", - apl: saleorApp.apl, - query: ProductDeletedDocument, - isActive: true, -}); - -const logger = createLogger({ - service: "webhook_product_deleted", -}); - -export const handler: NextWebhookApiHandler = async ( - req, - res, - context -) => { - await updateCacheOnWebhook({ - authData: context.authData, - channels: context.payload, - res, - }); -}; - -export default webhookProductDeleted.createHandler(handler); diff --git a/apps/products-feed/src/pages/api/webhooks/product_updated.ts b/apps/products-feed/src/pages/api/webhooks/product_updated.ts deleted file mode 100644 index 8c03561bf..000000000 --- a/apps/products-feed/src/pages/api/webhooks/product_updated.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { NextWebhookApiHandler, SaleorAsyncWebhook } from "@saleor/app-sdk/handlers/next"; -import { - ProductUpdatedDocument, - ProductWebhookPayloadFragment, -} from "../../../../generated/graphql"; -import { saleorApp } from "../../../saleor-app"; -import { createLogger } from "@saleor/apps-shared"; -import { updateCacheOnWebhook } from "../../../modules/metadata-cache/update-cache-on-webhook"; - -export const config = { - api: { - bodyParser: false, - }, -}; - -export const webhookProductUpdated = new SaleorAsyncWebhook({ - webhookPath: "api/webhooks/product_updated", - event: "PRODUCT_UPDATED", - apl: saleorApp.apl, - query: ProductUpdatedDocument, - isActive: true, -}); - -const logger = createLogger({ - service: "webhookProductUpdatedWebhookHandler", -}); - -export const handler: NextWebhookApiHandler = async ( - req, - res, - context -) => { - await updateCacheOnWebhook({ - authData: context.authData, - channels: context.payload, - res, - }); -}; - -export default webhookProductUpdated.createHandler(handler); diff --git a/apps/products-feed/src/pages/api/webhooks/product_variant_created.ts b/apps/products-feed/src/pages/api/webhooks/product_variant_created.ts deleted file mode 100644 index 30c2a547c..000000000 --- a/apps/products-feed/src/pages/api/webhooks/product_variant_created.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { NextWebhookApiHandler, SaleorAsyncWebhook } from "@saleor/app-sdk/handlers/next"; -import { - ProductVariantCreatedDocument, - ProductVariantWebhookPayloadFragment, -} from "../../../../generated/graphql"; -import { saleorApp } from "../../../saleor-app"; -import { createLogger } from "@saleor/apps-shared"; -import { updateCacheOnWebhook } from "../../../modules/metadata-cache/update-cache-on-webhook"; - -export const config = { - api: { - bodyParser: false, - }, -}; - -export const webhookProductVariantCreated = - new SaleorAsyncWebhook({ - webhookPath: "api/webhooks/product_variant_created", - event: "PRODUCT_VARIANT_CREATED", - apl: saleorApp.apl, - query: ProductVariantCreatedDocument, - isActive: true, - }); - -const logger = createLogger({ - service: "PRODUCT_VARIANT_CREATED webhook", -}); - -export const handler: NextWebhookApiHandler = async ( - req, - res, - context -) => { - await updateCacheOnWebhook({ - authData: context.authData, - channels: context.payload, - res, - }); -}; - -export default webhookProductVariantCreated.createHandler(handler); diff --git a/apps/products-feed/src/pages/api/webhooks/product_variant_deleted.ts b/apps/products-feed/src/pages/api/webhooks/product_variant_deleted.ts deleted file mode 100644 index 533b73fc6..000000000 --- a/apps/products-feed/src/pages/api/webhooks/product_variant_deleted.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { NextWebhookApiHandler, SaleorAsyncWebhook } from "@saleor/app-sdk/handlers/next"; -import { createLogger } from "@saleor/apps-shared"; -import { - ProductVariantDeletedDocument, - ProductVariantWebhookPayloadFragment, -} from "../../../../generated/graphql"; -import { saleorApp } from "../../../saleor-app"; -import { updateCacheOnWebhook } from "../../../modules/metadata-cache/update-cache-on-webhook"; - -export const config = { - api: { - bodyParser: false, - }, -}; - -export const webhookProductVariantDeleted = - new SaleorAsyncWebhook({ - webhookPath: "api/webhooks/product_variant_deleted", - event: "PRODUCT_VARIANT_DELETED", - apl: saleorApp.apl, - query: ProductVariantDeletedDocument, - isActive: true, - }); - -const logger = createLogger({ - service: "PRODUCT_VARIANT_DELETED", -}); - -export const handler: NextWebhookApiHandler = async ( - req, - res, - context -) => { - await updateCacheOnWebhook({ - authData: context.authData, - channels: context.payload, - res, - }); -}; - -export default webhookProductVariantDeleted.createHandler(handler); diff --git a/apps/products-feed/src/pages/api/webhooks/product_variant_updated.ts b/apps/products-feed/src/pages/api/webhooks/product_variant_updated.ts deleted file mode 100644 index f932bc9a1..000000000 --- a/apps/products-feed/src/pages/api/webhooks/product_variant_updated.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { NextWebhookApiHandler, SaleorAsyncWebhook } from "@saleor/app-sdk/handlers/next"; -import { createLogger } from "@saleor/apps-shared"; -import { - ProductVariantUpdatedDocument, - ProductVariantWebhookPayloadFragment, -} from "../../../../generated/graphql"; -import { saleorApp } from "../../../saleor-app"; -import { updateCacheOnWebhook } from "../../../modules/metadata-cache/update-cache-on-webhook"; - -export const config = { - api: { - bodyParser: false, - }, -}; - -export const webhookProductVariantUpdated = - new SaleorAsyncWebhook({ - webhookPath: "api/webhooks/product_variant_updated", - event: "PRODUCT_VARIANT_UPDATED", - apl: saleorApp.apl, - query: ProductVariantUpdatedDocument, - isActive: true, - }); - -const logger = createLogger({ - service: "webhookProductVariantUpdatedWebhookHandler", -}); - -export const handler: NextWebhookApiHandler = async ( - req, - res, - context -) => { - await updateCacheOnWebhook({ - authData: context.authData, - channels: context.payload, - res, - }); -}; - -export default webhookProductVariantUpdated.createHandler(handler); From 85f4830d5f12fe0fb54c2c3b9bd5e2d57f38225c Mon Sep 17 00:00:00 2001 From: Krzysztof Wolski Date: Fri, 22 Sep 2023 13:19:27 +0200 Subject: [PATCH 3/3] Add webhooks migration --- apps/products-feed/next.config.js | 9 ++++-- apps/products-feed/package.json | 2 ++ .../scripts/migrations/README.md | 7 +++++ .../scripts/migrations/migration-utils.ts | 20 +++++++++++++ .../run-webhooks-migration-dry-run.ts | 30 +++++++++++++++++++ .../migrations/run-webhooks-migration.ts | 30 +++++++++++++++++++ .../scripts/migrations/update-webhooks.ts | 29 ++++++++++++++++++ pnpm-lock.yaml | 6 ++++ 8 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 apps/products-feed/scripts/migrations/README.md create mode 100644 apps/products-feed/scripts/migrations/migration-utils.ts create mode 100644 apps/products-feed/scripts/migrations/run-webhooks-migration-dry-run.ts create mode 100644 apps/products-feed/scripts/migrations/run-webhooks-migration.ts create mode 100644 apps/products-feed/scripts/migrations/update-webhooks.ts diff --git a/apps/products-feed/next.config.js b/apps/products-feed/next.config.js index 6c8dd36d2..6e0aacdfb 100644 --- a/apps/products-feed/next.config.js +++ b/apps/products-feed/next.config.js @@ -6,7 +6,12 @@ const isSentryPropertiesInEnvironment = /** @type {import('next').NextConfig} */ const nextConfig = { reactStrictMode: true, - transpilePackages: ["@saleor/apps-shared", "@saleor/apps-ui", "@saleor/react-hook-form-macaw"], + transpilePackages: [ + "@saleor/apps-shared", + "@saleor/apps-ui", + "@saleor/react-hook-form-macaw", + "@saleor/webhook-utils", + ], }; const configWithSentry = withSentryConfig( @@ -22,7 +27,7 @@ const configWithSentry = withSentryConfig( tunnelRoute: "/monitoring", hideSourceMaps: true, disableLogger: true, - } + }, ); module.exports = isSentryPropertiesInEnvironment ? configWithSentry : nextConfig; diff --git a/apps/products-feed/package.json b/apps/products-feed/package.json index dd1e927d5..5bad2ed5b 100644 --- a/apps/products-feed/package.json +++ b/apps/products-feed/package.json @@ -19,6 +19,7 @@ "@saleor/apps-ui": "workspace:*", "@saleor/macaw-ui": "0.8.0-pre.127", "@saleor/react-hook-form-macaw": "workspace:*", + "@saleor/webhook-utils": "workspace:*", "@sentry/nextjs": "7.67.0", "@tanstack/react-query": "4.29.19", "@trpc/client": "10.38.1", @@ -27,6 +28,7 @@ "@trpc/server": "10.38.1", "@urql/exchange-auth": "^2.1.4", "@vitejs/plugin-react": "4.0.4", + "dotenv": "^16.3.1", "fast-xml-parser": "^4.0.15", "graphql": "16.7.1", "graphql-tag": "^2.12.6", diff --git a/apps/products-feed/scripts/migrations/README.md b/apps/products-feed/scripts/migrations/README.md new file mode 100644 index 000000000..248f04a1e --- /dev/null +++ b/apps/products-feed/scripts/migrations/README.md @@ -0,0 +1,7 @@ +# Webhook migration scripts + +Test migration with dry run, operation will not modify any data: +`npx tsx scripts/migrations/run-webhooks-migration-dry-run.ts` + +To start the migration run command: +`npx tsx scripts/migrations/run-webhooks-migration.ts` diff --git a/apps/products-feed/scripts/migrations/migration-utils.ts b/apps/products-feed/scripts/migrations/migration-utils.ts new file mode 100644 index 000000000..e0b1e433c --- /dev/null +++ b/apps/products-feed/scripts/migrations/migration-utils.ts @@ -0,0 +1,20 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +import { SaleorCloudAPL } from "@saleor/app-sdk/APL"; + +export const verifyRequiredEnvs = () => { + const requiredEnvs = ["SALEOR_CLOUD_TOKEN", "SALEOR_CLOUD_RESOURCE_URL"]; + + if (!requiredEnvs.every((env) => process.env[env])) { + throw new Error(`Missing envs: ${requiredEnvs.join(" | ")}`); + } +}; + +export const fetchCloudAplEnvs = () => { + const saleorAPL = new SaleorCloudAPL({ + token: process.env.SALEOR_CLOUD_TOKEN!, + resourceUrl: process.env.SALEOR_CLOUD_RESOURCE_URL!, + }); + + return saleorAPL.getAll(); +}; diff --git a/apps/products-feed/scripts/migrations/run-webhooks-migration-dry-run.ts b/apps/products-feed/scripts/migrations/run-webhooks-migration-dry-run.ts new file mode 100644 index 000000000..e71b6e735 --- /dev/null +++ b/apps/products-feed/scripts/migrations/run-webhooks-migration-dry-run.ts @@ -0,0 +1,30 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +import * as dotenv from "dotenv"; +import { fetchCloudAplEnvs, verifyRequiredEnvs } from "./migration-utils"; +import { updateWebhooksScript } from "./update-webhooks"; + +dotenv.config(); + +const runMigration = async () => { + console.log("Starting webhooks migration (dry run)"); + + verifyRequiredEnvs(); + + console.log("Envs verified, fetching envs"); + + const allEnvs = await fetchCloudAplEnvs().catch((r) => { + console.error("Could not fetch instances from the APL"); + console.error(r); + + process.exit(1); + }); + + for (const env of allEnvs) { + await updateWebhooksScript({ authData: env, dryRun: true }); + } + + console.log("Migration dry run complete"); +}; + +runMigration(); diff --git a/apps/products-feed/scripts/migrations/run-webhooks-migration.ts b/apps/products-feed/scripts/migrations/run-webhooks-migration.ts new file mode 100644 index 000000000..68cafa105 --- /dev/null +++ b/apps/products-feed/scripts/migrations/run-webhooks-migration.ts @@ -0,0 +1,30 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +import * as dotenv from "dotenv"; +import { fetchCloudAplEnvs, verifyRequiredEnvs } from "./migration-utils"; +import { updateWebhooksScript } from "./update-webhooks"; + +dotenv.config(); + +const runMigration = async () => { + console.log("Starting running migration"); + + verifyRequiredEnvs(); + + console.log("Envs verified, fetching envs"); + + const allEnvs = await fetchCloudAplEnvs().catch((r) => { + console.error("Could not fetch instances from the APL"); + console.error(r); + + process.exit(1); + }); + + for (const env of allEnvs) { + await updateWebhooksScript({ authData: env, dryRun: false }); + } + + console.log("Migration complete"); +}; + +runMigration(); diff --git a/apps/products-feed/scripts/migrations/update-webhooks.ts b/apps/products-feed/scripts/migrations/update-webhooks.ts new file mode 100644 index 000000000..786ceb497 --- /dev/null +++ b/apps/products-feed/scripts/migrations/update-webhooks.ts @@ -0,0 +1,29 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +import { createGraphQLClient } from "@saleor/apps-shared"; +import { AuthData } from "@saleor/app-sdk/APL"; +import { webhookMigrationRunner } from "@saleor/webhook-utils"; + +export const updateWebhooksScript = async ({ + authData, + dryRun, +}: { + authData: AuthData; + dryRun: boolean; +}) => { + console.log("Working on env: ", authData.saleorApiUrl); + + const client = createGraphQLClient({ + saleorApiUrl: authData.saleorApiUrl, + token: authData.token, + }); + + await webhookMigrationRunner({ + client, + dryRun, + getManifests: async ({ appDetails }) => { + // Products feed application has currently no webhooks, so we return empty array + return []; + }, + }); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8c02314ba..f85e82dc2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -923,6 +923,9 @@ importers: '@saleor/react-hook-form-macaw': specifier: workspace:* version: link:../../packages/react-hook-form-macaw + '@saleor/webhook-utils': + specifier: workspace:* + version: link:../../packages/webhook-utils '@sentry/nextjs': specifier: 7.67.0 version: 7.67.0(next@13.4.8)(react@18.2.0) @@ -947,6 +950,9 @@ importers: '@vitejs/plugin-react': specifier: 4.0.4 version: 4.0.4(vite@4.4.8) + dotenv: + specifier: ^16.3.1 + version: 16.3.1 fast-xml-parser: specifier: ^4.0.15 version: 4.0.15