diff --git a/package-lock.json b/package-lock.json index 553490b5830..1f6e4aa61fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.13", + "@rudderstack/integrations-lib": "^0.2.16", "@rudderstack/json-template-engine": "^0.19.5", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", @@ -6746,16 +6746,15 @@ } }, "node_modules/@rudderstack/integrations-lib": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.13.tgz", - "integrity": "sha512-MBI+OQpnYAuOzRlbGCnUX6oVfQsYA7daZ8z07WmqQYQtWFOfd2yFbaxKclu+R/a8W7+jBo4gvbW+ScEW6h+Mgg==", - "license": "MIT", + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.16.tgz", + "integrity": "sha512-wckZxn1EMu8nTV/sPmrWOTbKyC5WCM574q5q//B+AHhy68+c0pwvGq1nuSf2m+c6WaXgwUwxn28TcLc5w5Ga+g==", "dependencies": { "axios": "^1.4.0", - "axios-mock-adapter": "^1.22.0", "crypto": "^1.0.1", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-airbnb-typescript": "^17.1.0", + "fast-xml-parser": "^4.5.0", "get-value": "^3.0.1", "handlebars": "^4.7.8", "lodash": "^4.17.21", @@ -6764,9 +6763,22 @@ "set-value": "^4.1.0", "sha256": "^0.2.0", "tslib": "^2.4.0", + "uuid": "^11.0.5", "winston": "^3.11.0" } }, + "node_modules/@rudderstack/integrations-lib/node_modules/uuid": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.5.tgz", + "integrity": "sha512-508e6IcKLrhxKdBbcA2b4KQZlLVp2+J5UwQ6F7Drckkc5N9ZJwFa4TgWtsww9UG8fGHbm6gbV19TdM5pQ4GaIA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/@rudderstack/json-template-engine": { "version": "0.19.5", "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.19.5.tgz", @@ -8693,6 +8705,7 @@ }, "node_modules/axios-mock-adapter": { "version": "1.22.0", + "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -14238,6 +14251,7 @@ }, "node_modules/is-buffer": { "version": "2.0.5", + "dev": true, "funding": [ { "type": "github", diff --git a/package.json b/package.json index d320dd5c50d..e122044c1cd 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.13", + "@rudderstack/integrations-lib": "^0.2.16", "@rudderstack/json-template-engine": "^0.19.5", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", diff --git a/src/adapters/network.test.js b/src/adapters/network.test.js index 7894925ccd6..a4a2787504f 100644 --- a/src/adapters/network.test.js +++ b/src/adapters/network.test.js @@ -1,7 +1,20 @@ const mockLoggerInstance = { info: jest.fn(), }; -const { getFormData, httpPOST, httpGET, httpSend, fireHTTPStats } = require('./network'); +const { + getFormData, + httpPOST, + httpGET, + httpSend, + fireHTTPStats, + proxyRequest, + prepareProxyRequest, + handleHttpRequest, + httpDELETE, + httpPUT, + httpPATCH, + getPayloadData, +} = require('./network'); const { getFuncTestData } = require('../../test/testHelper'); jest.mock('../util/stats', () => ({ timing: jest.fn(), @@ -20,14 +33,28 @@ jest.mock('@rudderstack/integrations-lib', () => { }; }); -jest.mock('axios', () => jest.fn()); +// Mock the axios module +jest.mock('axios', () => { + const mockAxios = jest.fn(); // Mock the default axios function + mockAxios.get = jest.fn(); // Mock axios.get + mockAxios.post = jest.fn(); // Mock axios.post + mockAxios.put = jest.fn(); // Mock axios.put + mockAxios.patch = jest.fn(); // Mock axios.patch + mockAxios.delete = jest.fn(); // Mock axios.delete + + // Mock the axios.create method if needed + mockAxios.create = jest.fn(() => mockAxios); + + return mockAxios; // Return the mocked axios +}); + +const axios = require('axios'); jest.mock('../util/logger', () => ({ ...jest.requireActual('../util/logger'), getMatchedMetadata: jest.fn(), })); -const axios = require('axios'); const loggerUtil = require('../util/logger'); axios.post = jest.fn(); @@ -635,3 +662,338 @@ describe('logging in http methods', () => { expect(mockLoggerInstance.info).toHaveBeenCalledTimes(0); }); }); + +describe('httpDELETE tests', () => { + beforeEach(() => { + mockLoggerInstance.info.mockClear(); + loggerUtil.getMatchedMetadata.mockClear(); + axios.delete.mockClear(); + }); + + test('should call axios.delete with correct parameters and log request/response', async () => { + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/m/n/o', + requestMethod: 'delete', + }; + loggerUtil.getMatchedMetadata.mockReturnValue([statTags.metadata]); + + axios.delete.mockResolvedValueOnce({ + status: 200, + data: { a: 1, b: 2, c: 'abc' }, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + + await expect(httpDELETE('https://some.web.com/m/n/o', {}, statTags)).resolves.not.toThrow( + Error, + ); + expect(loggerUtil.getMatchedMetadata).toHaveBeenCalledTimes(2); + expect(mockLoggerInstance.info).toHaveBeenCalledTimes(2); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(1, ' [DT] /m/n/o request', { + body: undefined, + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + url: 'https://some.web.com/m/n/o', + method: 'delete', + }); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(2, ' [DT] /m/n/o response', { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + body: { a: 1, b: 2, c: 'abc' }, + status: 200, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + }); +}); + +describe('httpPUT tests', () => { + beforeEach(() => { + mockLoggerInstance.info.mockClear(); + loggerUtil.getMatchedMetadata.mockClear(); + axios.put.mockClear(); + }); + + test('should call axios.put with correct parameters and log request/response', async () => { + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/m/n/o', + requestMethod: 'put', + }; + loggerUtil.getMatchedMetadata.mockReturnValue([statTags.metadata]); + + axios.put.mockResolvedValueOnce({ + status: 200, + data: { a: 1, b: 2, c: 'abc' }, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + + await expect(httpPUT('https://some.web.com/m/n/o', {}, {}, statTags)).resolves.not.toThrow( + Error, + ); + expect(loggerUtil.getMatchedMetadata).toHaveBeenCalledTimes(2); + expect(mockLoggerInstance.info).toHaveBeenCalledTimes(2); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(1, ' [DT] /m/n/o request', { + body: {}, + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + url: 'https://some.web.com/m/n/o', + method: 'put', + }); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(2, ' [DT] /m/n/o response', { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + body: { a: 1, b: 2, c: 'abc' }, + status: 200, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + }); +}); + +describe('httpPATCH tests', () => { + beforeEach(() => { + mockLoggerInstance.info.mockClear(); + loggerUtil.getMatchedMetadata.mockClear(); + axios.patch.mockClear(); + }); + + test('should call axios.patch with correct parameters and log request/response', async () => { + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/m/n/o', + requestMethod: 'patch', + }; + loggerUtil.getMatchedMetadata.mockReturnValue([statTags.metadata]); + + axios.patch.mockResolvedValueOnce({ + status: 200, + data: { a: 1, b: 2, c: 'abc' }, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + + await expect(httpPATCH('https://some.web.com/m/n/o', {}, {}, statTags)).resolves.not.toThrow( + Error, + ); + expect(loggerUtil.getMatchedMetadata).toHaveBeenCalledTimes(2); + expect(mockLoggerInstance.info).toHaveBeenCalledTimes(2); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(1, ' [DT] /m/n/o request', { + body: {}, + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + url: 'https://some.web.com/m/n/o', + method: 'patch', + }); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(2, ' [DT] /m/n/o response', { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + body: { a: 1, b: 2, c: 'abc' }, + status: 200, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + }); +}); + +describe('getPayloadData tests', () => { + test('should return payload and payloadFormat for non-empty body', () => { + const body = { + JSON: { key: 'value' }, + XML: null, + FORM: null, + }; + const result = getPayloadData(body); + expect(result).toEqual({ payload: { key: 'value' }, payloadFormat: 'JSON' }); + }); + + test('should return undefined payload and payloadFormat for empty body', () => { + const body = {}; + const result = getPayloadData(body); + expect(result).toEqual({ payload: undefined, payloadFormat: undefined }); + }); +}); + +describe('prepareProxyRequest tests', () => { + test('should prepare proxy request with correct headers and payload', () => { + const request = { + body: { JSON: { key: 'value' } }, + method: 'POST', + params: { param1: 'value1' }, + endpoint: 'https://example.com', + headers: { 'Content-Type': 'application/json' }, + destinationConfig: { key: 'value' }, + }; + const result = prepareProxyRequest(request); + expect(result).toEqual({ + endpoint: 'https://example.com', + data: { key: 'value' }, + params: { param1: 'value1' }, + headers: { 'Content-Type': 'application/json', 'User-Agent': 'RudderLabs' }, + method: 'POST', + config: { key: 'value' }, + }); + }); +}); + +describe('handleHttpRequest tests', () => { + beforeEach(() => { + axios.post.mockClear(); + axios.get.mockClear(); + axios.put.mockClear(); + axios.patch.mockClear(); + axios.delete.mockClear(); + }); + + test('should handle POST request correctly', async () => { + axios.post.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('post', 'https://example.com', { key: 'value' }, {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle GET request correctly', async () => { + axios.get.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('get', 'https://example.com', {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle PUT request correctly', async () => { + axios.put.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('put', 'https://example.com', { key: 'value' }, {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle PATCH request correctly', async () => { + axios.patch.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('patch', 'https://example.com', { key: 'value' }, {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle DELETE request correctly', async () => { + axios.delete.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('delete', 'https://example.com', {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); +}); + +describe('proxyRequest tests', () => { + beforeEach(() => { + axios.mockClear(); + }); + + test('should proxy request correctly', async () => { + axios.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const request = { + body: { JSON: { key: 'value' } }, + method: 'POST', + params: { param1: 'value1' }, + endpoint: 'https://example.com', + headers: { 'Content-Type': 'application/json' }, + destinationConfig: { key: 'value' }, + metadata: { destType: 'DT' }, + }; + + const result = await proxyRequest(request, 'DT'); + expect(result).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + }); +}); diff --git a/src/middleware.test.js b/src/middleware.test.js new file mode 100644 index 00000000000..397dedfbd07 --- /dev/null +++ b/src/middleware.test.js @@ -0,0 +1,107 @@ +const Koa = require('koa'); // Import Koa +const { + addStatMiddleware, + addRequestSizeMiddleware, + getHeapProfile, + getCPUProfile, + initPyroscope, +} = require('./middleware'); + +const Pyroscope = require('@pyroscope/nodejs'); +const stats = require('./util/stats'); +const { getDestTypeFromContext } = require('@rudderstack/integrations-lib'); + +// Mock dependencies +jest.mock('@pyroscope/nodejs'); +jest.mock('./util/stats', () => ({ + timing: jest.fn(), + histogram: jest.fn(), +})); +jest.mock('@rudderstack/integrations-lib', () => ({ + getDestTypeFromContext: jest.fn(), +})); + +describe('Pyroscope Initialization', () => { + it('should initialize Pyroscope with the correct app name', () => { + initPyroscope(); + expect(Pyroscope.init).toHaveBeenCalledWith({ appName: 'rudder-transformer' }); + expect(Pyroscope.startHeapCollecting).toHaveBeenCalled(); + }); +}); + +describe('getCPUProfile', () => { + it('should call Pyroscope.collectCpu with the specified seconds', () => { + const seconds = 5; + getCPUProfile(seconds); + expect(Pyroscope.collectCpu).toHaveBeenCalledWith(seconds); + }); +}); + +describe('getHeapProfile', () => { + it('should call Pyroscope.collectHeap', () => { + getHeapProfile(); + expect(Pyroscope.collectHeap).toHaveBeenCalled(); + }); +}); + +describe('durationMiddleware', () => { + it('should record the duration of the request', async () => { + // Mock getDestTypeFromContext to return a fixed value + getDestTypeFromContext.mockReturnValue('mock-destination-type'); + + const app = new Koa(); // Create a Koa app instance + addStatMiddleware(app); // Pass the app instance to the middleware + + const ctx = { + method: 'GET', + status: 200, + request: { url: '/test' }, + }; + const next = jest.fn().mockResolvedValue(null); + + // Simulate the middleware execution + await app.middleware[0](ctx, next); + + expect(stats.timing).toHaveBeenCalledWith('http_request_duration', expect.any(Date), { + method: 'GET', + code: 200, + route: '/test', + destType: 'mock-destination-type', // Mocked value + }); + }); +}); + +describe('requestSizeMiddleware', () => { + it('should record the size of the request and response', async () => { + const app = new Koa(); // Create a Koa app instance + addRequestSizeMiddleware(app); // Pass the app instance to the middleware + + const ctx = { + method: 'POST', + status: 200, + request: { + url: '/test', + body: { key: 'value' }, + }, + response: { + body: { success: true }, + }, + }; + const next = jest.fn().mockResolvedValue(null); + + // Simulate the middleware execution + await app.middleware[0](ctx, next); + + expect(stats.histogram).toHaveBeenCalledWith('http_request_size', expect.any(Number), { + method: 'POST', + code: 200, + route: '/test', + }); + + expect(stats.histogram).toHaveBeenCalledWith('http_response_size', expect.any(Number), { + method: 'POST', + code: 200, + route: '/test', + }); + }); +}); diff --git a/src/routerUtils.js b/src/routerUtils.js index 081070d78ac..67b6e0d31c4 100644 --- a/src/routerUtils.js +++ b/src/routerUtils.js @@ -4,13 +4,7 @@ const logger = require('./logger'); const { proxyRequest } = require('./adapters/network'); const { nodeSysErrorToStatus } = require('./adapters/utils/networkUtils'); -let areFunctionsEnabled = -1; -const functionsEnabled = () => { - if (areFunctionsEnabled === -1) { - areFunctionsEnabled = process.env.ENABLE_FUNCTIONS === 'false' ? 0 : 1; - } - return areFunctionsEnabled === 1; -}; +const functionsEnabled = () => process.env.ENABLE_FUNCTIONS !== 'false'; const userTransformHandler = () => { if (functionsEnabled()) { diff --git a/src/routerUtils.test.js b/src/routerUtils.test.js new file mode 100644 index 00000000000..81c8ed49919 --- /dev/null +++ b/src/routerUtils.test.js @@ -0,0 +1,126 @@ +const { sendToDestination, userTransformHandler } = require('./routerUtils'); // Update the path accordingly + +const logger = require('./logger'); +const { proxyRequest } = require('./adapters/network'); +const { nodeSysErrorToStatus } = require('./adapters/utils/networkUtils'); + +// Mock dependencies +jest.mock('./logger'); +jest.mock('./adapters/network'); +jest.mock('./adapters/utils/networkUtils'); + +describe('sendToDestination', () => { + beforeEach(() => { + jest.clearAllMocks(); // Clear mocks before each test + }); + + it('should send a request to the destination and return a successful response', async () => { + // Mock proxyRequest to return a successful response + proxyRequest.mockResolvedValue({ + success: true, + response: { + headers: { 'content-type': 'application/json' }, + data: { message: 'Success' }, + status: 200, + }, + }); + + const destination = 'mock-destination'; + const payload = { key: 'value' }; + + const result = await sendToDestination(destination, payload); + + expect(logger.info).toHaveBeenCalledWith('Request recieved for destination', destination); + expect(proxyRequest).toHaveBeenCalledWith(payload); + expect(result).toEqual({ + headers: { 'content-type': 'application/json' }, + response: { message: 'Success' }, + status: 200, + }); + }); + + it('should handle network failure and return a parsed response', async () => { + // Mock proxyRequest to return a network failure + proxyRequest.mockResolvedValue({ + success: false, + response: { + code: 'ENOTFOUND', // Simulate a network error + }, + }); + + // Mock nodeSysErrorToStatus to return a specific error message and status + nodeSysErrorToStatus.mockReturnValue({ + message: 'Network error', + status: 500, + }); + + const destination = 'mock-destination'; + const payload = { key: 'value' }; + + const result = await sendToDestination(destination, payload); + + expect(logger.info).toHaveBeenCalledWith('Request recieved for destination', destination); + expect(proxyRequest).toHaveBeenCalledWith(payload); + expect(nodeSysErrorToStatus).toHaveBeenCalledWith('ENOTFOUND'); + expect(result).toEqual({ + headers: null, + networkFailure: true, + response: 'Network error', + status: 500, + }); + }); + + it('should handle axios error with response and return a parsed response', async () => { + // Mock proxyRequest to return an axios error with response + proxyRequest.mockResolvedValue({ + success: false, + response: { + response: { + headers: { 'content-type': 'application/json' }, + status: 400, + data: 'Bad Request', + }, + }, + }); + + const destination = 'mock-destination'; + const payload = { key: 'value' }; + + const result = await sendToDestination(destination, payload); + + expect(logger.info).toHaveBeenCalledWith('Request recieved for destination', destination); + expect(proxyRequest).toHaveBeenCalledWith(payload); + expect(result).toEqual({ + headers: { 'content-type': 'application/json' }, + status: 400, + response: 'Bad Request', + }); + }); +}); + +describe('userTransformHandler', () => { + beforeEach(() => { + jest.clearAllMocks(); // Clear mocks before each test + jest.resetModules(); // Reset modules to reset process.env + }); + + it('should return userTransformHandler when functions are enabled', () => { + // Mock process.env to enable functions + process.env.ENABLE_FUNCTIONS = 'true'; + + const mockUserTransformHandler = jest.fn(); + jest.mock('./util/customTransformer', () => ({ + userTransformHandler: mockUserTransformHandler, + })); + + const result = userTransformHandler(); + expect(result).toBe(mockUserTransformHandler); + }); + + it('should throw an error when functions are not enabled', () => { + // Mock process.env to disable functions + process.env.ENABLE_FUNCTIONS = 'false'; + + expect(() => userTransformHandler()).toThrow('Functions are not enabled'); + }); +}); diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 18cbcdf6b7e..86a6bff2644 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -445,12 +445,6 @@ class Prometheus { type: 'counter', labelNames: ['event', 'writeKey'], }, - { - name: 'shopify_pixel_cart_token_not_found_server_side', - help: 'shopify_pixel_cart_token_not_found_server_side', - type: 'counter', - labelNames: ['event', 'writeKey'], - }, { name: 'shopify_pixel_cart_token_set', help: 'shopify_pixel_cart_token_set', @@ -463,6 +457,12 @@ class Prometheus { type: 'counter', labelNames: ['event', 'writeKey'], }, + { + name: 'shopify_pixel_id_stitch_gaps', + help: 'shopify_pixel_id_stitch_gaps', + type: 'counter', + labelNames: ['event', 'reason', 'source', 'writeKey'], + }, { name: 'outgoing_request_count', help: 'Outgoing HTTP requests count', diff --git a/src/v0/destinations/adobe_analytics/utils.test.js b/src/v0/destinations/adobe_analytics/utils.test.js new file mode 100644 index 00000000000..df40bf2ff31 --- /dev/null +++ b/src/v0/destinations/adobe_analytics/utils.test.js @@ -0,0 +1,271 @@ +const { + handleContextData, + handleEvar, + handleHier, + handleList, + handleCustomProperties, + stringifyValueAndJoinWithDelimiter, + escapeToHTML, +} = require('./utils'); // Update the path accordingly + +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + +describe('handleContextData', () => { + it('should add context data to the payload when values are found', () => { + const payload = {}; + const destinationConfig = { + contextDataPrefix: 'c_', + contextDataMapping: { + 'user.id': 'userId', + 'user.email': 'userEmail', + }, + }; + const message = { + user: { + id: '123', + email: 'test@example.com', + }, + }; + + const result = handleContextData(payload, destinationConfig, message); + + expect(result.contextData).toEqual({ + c_userId: '123', + c_userEmail: 'test@example.com', + }); + }); + + it('should not add context data to the payload when no values are found', () => { + const payload = {}; + const destinationConfig = { + contextDataPrefix: 'c_', + contextDataMapping: { + 'user.id': 'userId', + 'user.email': 'userEmail', + }, + }; + const message = { + user: { + name: 'John Doe', + }, + }; + + const result = handleContextData(payload, destinationConfig, message); + + expect(result.contextData).toBeUndefined(); + }); +}); + +describe('handleEvar', () => { + it('should map properties to eVars in the payload', () => { + const payload = {}; + const destinationConfig = { + eVarMapping: { + productId: '1', + category: '2', + }, + }; + const message = { + properties: { + productId: 'p123', + category: 'electronics', + }, + }; + + const result = handleEvar(payload, destinationConfig, message); + + expect(result).toEqual({ + eVar1: 'p123', + eVar2: 'electronics', + }); + }); + + it('should not add eVars to the payload when no values are found', () => { + const payload = {}; + const destinationConfig = { + eVarMapping: { + productId: '1', + category: '2', + }, + }; + const message = { + properties: { + name: 'Product Name', + }, + }; + + const result = handleEvar(payload, destinationConfig, message); + + expect(result).toEqual({}); + }); +}); + +describe('handleHier', () => { + it('should map properties to hVars in the payload', () => { + const payload = {}; + const destinationConfig = { + hierMapping: { + section: '1', + subsection: '2', + }, + }; + const message = { + properties: { + section: 'home', + subsection: 'kitchen', + }, + }; + + const result = handleHier(payload, destinationConfig, message); + + expect(result).toEqual({ + hier1: 'home', + hier2: 'kitchen', + }); + }); + + it('should not add hVars to the payload when no values are found', () => { + const payload = {}; + const destinationConfig = { + hierMapping: { + section: '1', + subsection: '2', + }, + }; + const message = { + properties: { + name: 'Section Name', + }, + }; + + const result = handleHier(payload, destinationConfig, message); + + expect(result).toEqual({}); + }); +}); + +describe('handleList', () => { + it('should map properties to list variables in the payload', () => { + const payload = {}; + const destinationConfig = { + listMapping: { + products: '1', + }, + listDelimiter: { + products: ',', + }, + }; + const message = { + properties: { + products: ['p1', 'p2', 'p3'], + }, + }; + + const result = handleList(payload, destinationConfig, message); + + expect(result).toEqual({ + list1: 'p1,p2,p3', + }); + }); + + it('should throw an error when list properties are not strings or arrays', () => { + const payload = {}; + const destinationConfig = { + listMapping: { + products: '1', + }, + listDelimiter: { + products: ',', + }, + }; + const message = { + properties: { + products: 123, // Invalid type + }, + }; + + expect(() => handleList(payload, destinationConfig, message)).toThrow(InstrumentationError); + }); +}); + +describe('handleCustomProperties', () => { + it('should map properties to custom properties in the payload', () => { + const payload = {}; + const destinationConfig = { + customPropsMapping: { + color: '1', + size: '2', + }, + propsDelimiter: { + color: ',', + size: ';', + }, + }; + const message = { + properties: { + color: 'red,green,blue', + size: ['S', 'M', 'L'], + }, + }; + + const result = handleCustomProperties(payload, destinationConfig, message); + + expect(result).toEqual({ + prop1: 'red,green,blue', + prop2: 'S;M;L', + }); + }); + + it('should throw an error when custom properties are not strings or arrays', () => { + const payload = {}; + const destinationConfig = { + customPropsMapping: { + color: '1', + }, + propsDelimiter: { + color: ',', + }, + }; + const message = { + properties: { + color: 123, // Invalid type + }, + }; + + expect(() => handleCustomProperties(payload, destinationConfig, message)).toThrow( + InstrumentationError, + ); + }); +}); + +describe('stringifyValueAndJoinWithDelimiter', () => { + it('should join values with a delimiter after stringifying them', () => { + const values = [1, null, 'test', true]; + const result = stringifyValueAndJoinWithDelimiter(values, '|'); + + expect(result).toBe('1|null|test|true'); + }); + + it('should use the default delimiter if none is provided', () => { + const values = [1, 2, 3]; + const result = stringifyValueAndJoinWithDelimiter(values); + + expect(result).toBe('1;2;3'); + }); +}); + +describe('escapeToHTML', () => { + it('should escape HTML entities in a string', () => { + const input = '
&
'; + const result = escapeToHTML(input); + + expect(result).toBe('<div>&</div>'); + }); + + it('should return non-string values unchanged', () => { + const input = 123; + const result = escapeToHTML(input); + + expect(result).toBe(123); + }); +}); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js index 1fe92bbee0d..6d893a0c040 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -19,6 +19,7 @@ const { getProductsFromLineItems, setAnonymousId, handleCommonProperties, + addCartTokenHashToTraits, } = require('./serverSideUtlis'); const NO_OPERATION_SUCCESS = { @@ -113,6 +114,8 @@ const processEvent = async (inputEvent, metricMetadata) => { } // attach userId, email and other contextual properties message = handleCommonProperties(message, event, shopifyTopic); + // add cart_token_hash to traits if cart_token is present + message = addCartTokenHashToTraits(message, event); message = removeUndefinedAndNullValues(message); return message; }; diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js index 070fdafdd71..15e16c4c196 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -4,11 +4,14 @@ const { createPropertiesForEcomEventFromWebhook, getAnonymousIdFromAttributes, getCartToken, + setAnonymousId, + addCartTokenHashToTraits, } = require('./serverSideUtlis'); const { RedisDB } = require('../../../../util/redis/redisConnector'); const { lineItemsMappingJSON } = require('../../../../v0/sources/shopify/config'); const Message = require('../../../../v0/sources/message'); +const { property } = require('lodash'); jest.mock('../../../../v0/sources/message'); const LINEITEMS = [ @@ -131,7 +134,7 @@ describe('serverSideUtils.js', () => { }); }); - describe('getCartToken', () => { + describe('Test getCartToken', () => { it('should return null if cart_token is not present', () => { const event = {}; const result = getCartToken(event); @@ -144,6 +147,31 @@ describe('serverSideUtils.js', () => { expect(result).toEqual('cartTokenTest1'); }); }); + + describe('Test addCartTokenHashToTraits', () => { + // Add cart token hash to traits when cart token exists in event + it('should add cart_token_hash to message traits when cart token exists', () => { + const message = { traits: { existingTrait: 'value' } }; + const event = { cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ' }; + const expectedHash = '9125e1da-57b9-5bdc-953e-eb2b0ded5edc'; + + addCartTokenHashToTraits(message, event); + + expect(message.traits).toEqual({ + existingTrait: 'value', + cart_token_hash: expectedHash, + }); + }); + + // Do not add cart token hash to traits when cart token does not exist in event + it('should not add cart_token_hash to message traits when cart token does not exist', () => { + const message = { traits: { existingTrait: 'value' } }; + const event = { property: 'value' }; + addCartTokenHashToTraits(message, event); + + expect(message.traits).toEqual({ existingTrait: 'value' }); + }); + }); }); describe('Redis cart token tests', () => { @@ -170,4 +198,22 @@ describe('Redis cart token tests', () => { expect(getValSpy).toHaveBeenCalledWith('pixel:cartTokenTest1'); expect(message.anonymousId).toEqual('anonymousIdTest1'); }); + + it('should generate new anonymousId using UUID v5 when no existing ID is found', async () => { + const message = {}; + const event = { + note_attributes: [], + }; + const metricMetadata = { source: 'test', writeKey: 'test-key' }; + const cartToken = 'test-cart-token'; + const mockRedisData = null; + const expectedAnonymousId = '40a532a2-88be-5e3a-8687-56e34739e89d'; + jest.mock('uuid', () => ({ + v5: jest.fn(() => expectedAnonymousId), + DNS: 'dns-namespace', + })); + RedisDB.getVal = jest.spyOn(RedisDB, 'getVal').mockResolvedValue(mockRedisData); + await setAnonymousId(message, { ...event, cart_token: cartToken }, metricMetadata); + expect(message.anonymousId).toBe(expectedAnonymousId); + }); }); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index 0d81de99ac3..e6da784b455 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -1,6 +1,6 @@ /* eslint-disable no-param-reassign */ const get = require('get-value'); -const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); +const { isDefinedAndNotNull, uuidv5 } = require('@rudderstack/integrations-lib'); const { extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); const { constructPayload } = require('../../../../v0/util'); const { INTEGERATION, lineItemsMappingJSON, productMappingJSON } = require('../config'); @@ -65,6 +65,23 @@ const getAnonymousIdFromAttributes = (event) => { */ const getCartToken = (event) => event?.cart_token || null; +/** + * Adds the cartTokenHash to the traits object in the message + * @param {Object} message + * @param {String} event + * */ +const addCartTokenHashToTraits = (message, event) => { + const cartToken = getCartToken(event); + if (cartToken) { + const cartTokenHash = uuidv5(cartToken); + message.traits = { + ...message.traits, + cart_token_hash: cartTokenHash, + }; + } + return message; +}; + /** * Handles the anonymousId assignment for the message, based on the event attributes and redis data * @param {Object} message rudderstack message object @@ -73,22 +90,36 @@ const getCartToken = (event) => event?.cart_token || null; */ const setAnonymousId = async (message, event, metricMetadata) => { const anonymousId = getAnonymousIdFromAttributes(event); + const cartToken = getCartToken(event); + const cartTokenHash = cartToken ? uuidv5(cartToken) : null; if (isDefinedAndNotNull(anonymousId)) { message.anonymousId = anonymousId; - } else { - // if anonymousId is not present in note_attributes or note_attributes is not present, query redis for anonymousId - const cartToken = getCartToken(event); - if (cartToken) { - const redisData = await RedisDB.getVal(`pixel:${cartToken}`); - if (redisData?.anonymousId) { - message.anonymousId = redisData.anonymousId; - } + } + // if anonymousId is not present in note_attributes or note_attributes is not present, query redis for anonymousId + // when cart_token is present + else if (cartToken) { + const redisData = await RedisDB.getVal(`pixel:${cartToken}`); + if (redisData?.anonymousId) { + message.anonymousId = redisData.anonymousId; } else { - stats.increment('shopify_pixel_cart_token_not_found_server_side', { + // if anonymousId is not present in note_attributes or redis, generate a new anonymousId + // the anonymousId will be generated by hashing the cart_token using uuidv5 + // this hash will be present in the traits object as cart_token_hash + message.anonymousId = cartTokenHash; + stats.increment('shopify_pixel_id_stitch_gaps', { + event: message.event, + reason: 'redis_cache_miss', source: metricMetadata.source, writeKey: metricMetadata.writeKey, }); } + } else { + stats.increment('shopify_pixel_id_stitch_gaps', { + event: message.event, + reason: 'cart_token_miss', + source: metricMetadata.source, + writeKey: metricMetadata.writeKey, + }); } }; @@ -138,4 +169,5 @@ module.exports = { getAnonymousIdFromAttributes, setAnonymousId, handleCommonProperties, + addCartTokenHashToTraits, }; diff --git a/test/integrations/sources/shopify/mocks.ts b/test/integrations/sources/shopify/mocks.ts index e1895e78124..9a3fe2f989b 100644 --- a/test/integrations/sources/shopify/mocks.ts +++ b/test/integrations/sources/shopify/mocks.ts @@ -1,5 +1,7 @@ import utils from '../../../../src/v0/util'; +import { RedisDB } from '../../../../src/util/redis/redisConnector'; export const mockFns = (_) => { jest.spyOn(utils, 'generateUUID').mockReturnValue('5d3e2cb6-4011-5c9c-b7ee-11bc1e905097'); + jest.spyOn(RedisDB, 'getVal').mockResolvedValue({}); }; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts index 4938294ef99..19e66f450a5 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -1,5 +1,6 @@ // This file contains the test scenarios for the server-side events from the Shopify GraphQL API for // the v1 transformation flow +import { mockFns } from '../mocks'; import { dummySourceConfig, note_attributes } from '../constants'; export const checkoutEventsTestScenarios = [ @@ -100,11 +101,6 @@ export const checkoutEventsTestScenarios = [ }, }, source: dummySourceConfig, - query_parameters: { - topic: ['carts_update'], - writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], - version: ['pixel'], - }, }, ], method: 'POST', @@ -240,6 +236,7 @@ export const checkoutEventsTestScenarios = [ timestamp: '2024-11-06T02:22:02.000Z', traits: { shippingAddress: [], + cart_token_hash: '9125e1da-57b9-5bdc-953e-eb2b0ded5edc', }, anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', }, @@ -576,6 +573,7 @@ export const checkoutEventsTestScenarios = [ province_code: 'AZ', zip: '85003', }, + cart_token_hash: '9e189f39-da46-58df-81b4-5e507d9ef64e', adminGraphqlApiId: 'gid://shopify/Customer/7188389789809', currency: 'USD', email: 'testuser101@gmail.com', @@ -1258,6 +1256,7 @@ export const checkoutEventsTestScenarios = [ country_name: 'United States', default: true, }, + cart_token_hash: '9125e1da-57b9-5bdc-953e-eb2b0ded5edc', state: 'disabled', verifiedEmail: true, taxExempt: false, @@ -1654,6 +1653,7 @@ export const checkoutEventsTestScenarios = [ country_name: 'United States', default: true, }, + cart_token_hash: '9125e1da-57b9-5bdc-953e-eb2b0ded5edc', state: 'disabled', currency: 'USD', taxExemptions: [], @@ -1838,4 +1838,248 @@ export const checkoutEventsTestScenarios = [ }, }, }, -]; + { + id: 'c001', + name: 'shopify', + description: + 'Track Call -> Checkout Started event from Pixel app, with no anonymoudId in redis. anonymousId is set as hash of cart_token (race condition scenario)', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 35550298931313, + token: '84ad78572dae52a8cbea7d55371afe89', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + email: null, + gateway: null, + buyer_accepts_marketing: false, + buyer_accepts_sms_marketing: false, + sms_marketing_phone: null, + created_at: '2024-11-06T02:22:00+00:00', + updated_at: '2024-11-05T21:22:02-05:00', + landing_site: '/', + note: '', + note_attributes: [], + referring_site: '', + shipping_lines: [], + shipping_address: [], + taxes_included: false, + total_weight: 0, + currency: 'USD', + completed_at: null, + phone: null, + customer_locale: 'en-US', + line_items: [ + { + key: '41327142600817', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + presentment_variant_title: '', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '600.00', + price: '600.00', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35550298931313', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ/recover?key=0385163be3875d3a2117e982d9cc3517&locale=en-US', + discount_codes: [], + tax_lines: [], + presentment_currency: 'USD', + source_name: 'web', + total_line_items_price: '600.00', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '600.00', + total_price: '600.00', + total_duties: '0.00', + device_id: null, + user_id: null, + location_id: null, + source_identifier: null, + source_url: null, + source: null, + closed_at: null, + query_parameters: { + topic: ['checkouts_create'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'checkouts_create', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + shopifyDetails: { + id: 35550298931313, + token: '84ad78572dae52a8cbea7d55371afe89', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + email: null, + gateway: null, + buyer_accepts_marketing: false, + buyer_accepts_sms_marketing: false, + sms_marketing_phone: null, + created_at: '2024-11-06T02:22:00+00:00', + updated_at: '2024-11-05T21:22:02-05:00', + landing_site: '/', + note: '', + note_attributes: [], + referring_site: '', + shipping_lines: [], + shipping_address: [], + taxes_included: false, + total_weight: 0, + currency: 'USD', + completed_at: null, + phone: null, + customer_locale: 'en-US', + line_items: [ + { + key: '41327142600817', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + presentment_variant_title: '', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '600.00', + price: '600.00', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35550298931313', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ/recover?key=0385163be3875d3a2117e982d9cc3517&locale=en-US', + discount_codes: [], + tax_lines: [], + presentment_currency: 'USD', + source_name: 'web', + total_line_items_price: '600.00', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '600.00', + total_price: '600.00', + total_duties: '0.00', + device_id: null, + user_id: null, + location_id: null, + source_identifier: null, + source_url: null, + source: null, + closed_at: null, + }, + }, + integrations: { + SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, + }, + type: 'track', + event: 'Checkout Started Webhook', + properties: { + order_id: '35550298931313', + value: 600, + tax: 0, + currency: 'USD', + products: [ + { + product_id: '7234590408817', + price: 600.0, + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + timestamp: '2024-11-06T02:22:02.000Z', + traits: { + shippingAddress: [], + cart_token_hash: '9125e1da-57b9-5bdc-953e-eb2b0ded5edc', + }, + anonymousId: '9125e1da-57b9-5bdc-953e-eb2b0ded5edc', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d1) => ({ ...d1, mockFns })); diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts index 422fe0135a2..8385122b222 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -556,6 +556,7 @@ export const genericTrackTestScenarios = [ }, traits: { email: 'henry@wfls.com', + cart_token_hash: '9125e1da-57b9-5bdc-953e-eb2b0ded5edc', }, anonymousId: '50ead33e-d763-4854-b0ab-765859ef05cb', },