Merge pull request #1825 from rudderlabs/master

chore: sync master to rudder-saas
rudderlabs · Jan 25, 2023 · 73135b8 · 73135b8
2 parents d0c3536 + a49be9e
commit 73135b8
Show file tree

Hide file tree

Showing 19 changed files with 887 additions and 40 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.12.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.11.1...v1.12.0) (2023-01-19)
+
+
+### Features
+
+* **BQStream:** add batch support ([#1377](https://github.com/rudderlabs/rudder-transformer/issues/1377)) ([14c7531](https://github.com/rudderlabs/rudder-transformer/commit/14c7531635b5348ef518dcad483f25d4adeddddd))
+* **destination:** onboard lemnisk integration  ([#1787](https://github.com/rudderlabs/rudder-transformer/issues/1787)) ([3c6b9e1](https://github.com/rudderlabs/rudder-transformer/commit/3c6b9e148dff559357fb61de49602f9d1689d699))
+* ga4 page calls are discarded if hybrid mode is enabled ([#1794](https://github.com/rudderlabs/rudder-transformer/issues/1794)) ([ca12d07](https://github.com/rudderlabs/rudder-transformer/commit/ca12d078e3f936c4c0fd4449259d1a55ba0a4424))
+* sessionId consistency across destinations ([#1789](https://github.com/rudderlabs/rudder-transformer/issues/1789)) ([ff68a44](https://github.com/rudderlabs/rudder-transformer/commit/ff68a4488e50f4a44c950395d0f9e5dc514db1df))
+
+
+### Bug Fixes
+
+* add sources as valid channel to cdkv1 ([bd74ef7](https://github.com/rudderlabs/rudder-transformer/commit/bd74ef7eff712d4db75856a205ddaa473d80ddd9))
+* add sources as valid channel to cdkv1 ([#1803](https://github.com/rudderlabs/rudder-transformer/issues/1803)) ([e3057db](https://github.com/rudderlabs/rudder-transformer/commit/e3057dbff9d8daa1f64b5cd6de6b57ab97c016ee))
+* array type check for externalIdArray ([#1785](https://github.com/rudderlabs/rudder-transformer/issues/1785)) ([dec3bb6](https://github.com/rudderlabs/rudder-transformer/commit/dec3bb6661b6737203964b2c4b5e3d2bd7421433))
+* change destination configuration errors to abortable ([#1790](https://github.com/rudderlabs/rudder-transformer/issues/1790)) ([fb1281d](https://github.com/rudderlabs/rudder-transformer/commit/fb1281d2bc090bda34c7420c10946504e83756ef))
+* **integration:** GAOC - fix timestamp format, allow calls without custom variables ([#1796](https://github.com/rudderlabs/rudder-transformer/issues/1796)) ([7c450ee](https://github.com/rudderlabs/rudder-transformer/commit/7c450ee78db2052bbb70866cbc6bd98cfd9c32b4))
+* iterable alias call is separated from identify batching ([#1777](https://github.com/rudderlabs/rudder-transformer/issues/1777)) ([3676c45](https://github.com/rudderlabs/rudder-transformer/commit/3676c4591e8b241ad6a7873954bc8f07e7a69584))
+* products array mapping and rename impact_radius to impact ([#1797](https://github.com/rudderlabs/rudder-transformer/issues/1797)) ([f812f0d](https://github.com/rudderlabs/rudder-transformer/commit/f812f0d3fbff6d6bfdd3670c59cf8ea01744f80f))
+* remove regex validation for phone_number ([#1771](https://github.com/rudderlabs/rudder-transformer/issues/1771)) ([6c01642](https://github.com/rudderlabs/rudder-transformer/commit/6c016428b496cea7e3771d3cf5ab4dfbbd7e382b))
+
+
+### Miscellaneous
+
+* fix formatting and linting issues ([#1772](https://github.com/rudderlabs/rudder-transformer/issues/1772)) ([fb8b818](https://github.com/rudderlabs/rudder-transformer/commit/fb8b818b2cbd05f784117b9f3040856dab1a7346))
+* **owner:** added a code owner ([#1793](https://github.com/rudderlabs/rudder-transformer/issues/1793)) ([1c6f92a](https://github.com/rudderlabs/rudder-transformer/commit/1c6f92a90363489bfc3e9430ec50fb30b2a65557))
+
 ## [1.11.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.11.0...v1.11.1) (2023-01-13)
 
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "rudder-transformer",
-  "version": "1.11.1",
+  "version": "1.12.0",
   "description": "",
   "homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
   "bugs": {

diff --git a/src/v0/destinations/rockerbox/transform.js b/src/v0/destinations/rockerbox/transform.js
@@ -5,6 +5,7 @@ const {
   constructPayload,
   simpleProcessRouterDest,
   getHashFromArray,
+  isDefinedAndNotNullAndNotEmpty,
 } = require('../../util');
 const { EventType } = require('../../../constants');
 const { CONFIG_CATEGORIES, MAPPING_CONFIG } = require('./config');
@@ -15,10 +16,21 @@ const responseBuilderSimple = (message, category, destination) => {
   // conversion_source is explicitly set to RudderStack
   payload.conversion_source = 'RudderStack';
 
-  const { advertiserId, eventsMap } = destination.Config;
+  const { advertiserId, eventsMap, customPropsMapping } = destination.Config;
   // we will map the events to their rockerbox counterparts from UI
   const eventsHashMap = getHashFromArray(eventsMap);
 
+  //handle custom properties mapped in the UI
+  const customPropsHashMap = getHashFromArray(customPropsMapping);
+
+  if(isDefinedAndNotNullAndNotEmpty(customPropsHashMap)){
+    for (const key in customPropsHashMap) {
+      // check if the custom property is passed in event properties
+      if (isDefinedAndNotNullAndNotEmpty(message.properties[key]))
+        payload[customPropsHashMap[key]] = message.properties[key]
+    }
+  }
+
   // Reject other unmapped events
   if (!eventsHashMap[message.event.toLowerCase()]) {
     throw new ConfigurationError('The event is not associated to a RockerBox event. Aborting!');

diff --git a/src/v0/destinations/salesforce/transform.js b/src/v0/destinations/salesforce/transform.js
@@ -114,7 +114,7 @@ async function getSaleforceIdForRecord(
   if (processedsfSearchResponse.status !== 200) {
     salesforceResponseHandler(
       processedsfSearchResponse,
-      `SALESFORCE SEARCH BY ID: ${JSON.stringify(processedsfSearchResponse.response)}`,
+      `:- SALESFORCE SEARCH BY ID`,
       destination.ID,
     );
   }
@@ -212,7 +212,7 @@ async function getSalesforceIdFromPayload(message, authorizationData, destinatio
     if (processedLeadQueryResponse.status !== 200) {
       salesforceResponseHandler(
         processedLeadQueryResponse,
-        `During Lead Query: ${JSON.stringify(processedLeadQueryResponse.response)}`,
+        `:- during Lead Query`,
         destination.ID,
       );
     }

diff --git a/src/v0/destinations/salesforce/utils.js b/src/v0/destinations/salesforce/utils.js
@@ -25,27 +25,31 @@ const salesforceResponseHandler = (destResponse, sourceMessage, authKey) => {
   // if the response from destination is not a success case build an explicit error
   if (!isHttpStatusSuccess(status) && status >= 400) {
     const matchErrorCode = (errorCode) =>
-      response && Array.isArray(response) && response.some((resp) => resp.errorCode === errorCode);
+      response && Array.isArray(response) && response.some((resp) => resp?.errorCode === errorCode);
     if (status === 401 && authKey && matchErrorCode('INVALID_SESSION_ID')) {
       // checking for invalid/expired token errors and evicting cache in that case
       // rudderJobMetadata contains some destination info which is being used to evict the cache
       ACCESS_TOKEN_CACHE.del(authKey);
       throw new RetryableError(
-        `${DESTINATION} Request Failed - due to ${response[0].message}, (Retryable).${sourceMessage}`,
+        `${DESTINATION} Request Failed - due to "INVALID_SESSION_ID", (Retryable) ${sourceMessage}`,
         500,
         destResponse,
       );
     } else if (status === 403 && matchErrorCode('REQUEST_LIMIT_EXCEEDED')) {
       // If the error code is REQUEST_LIMIT_EXCEEDED, you’ve exceeded API request limits in your org.
       throw new ThrottledError(
-        `${DESTINATION} Request Failed - due to ${response[0].message}, (Throttled).${sourceMessage}`,
+        `${DESTINATION} Request Failed - due to "REQUEST_LIMIT_EXCEEDED", (Throttled) ${sourceMessage}`,
         destResponse,
       );
-    } else if (status === 503) {
+    } else if (status === 503 || status === 500) {
       // The salesforce server is unavailable to handle the request. Typically this occurs if the server is down
       // for maintenance or is currently overloaded.
       throw new RetryableError(
-        `${DESTINATION} Request Failed - due to ${response[0].message}, (Retryable).${sourceMessage}`,
+        `${DESTINATION} Request Failed - due to "${
+          response && Array.isArray(response) && response[0]?.message?.length > 0
+            ? response[0].message
+            : JSON.stringify(response)
+        }", (Retryable) ${sourceMessage}`,
         500,
         destResponse,
       );
@@ -55,9 +59,11 @@ const salesforceResponseHandler = (destResponse, sourceMessage, authKey) => {
     if (response && Array.isArray(response)) {
       errorMessage = response[0].message;
     }
-
+    // aborting for all other error codes
     throw new AbortedError(
-      `${DESTINATION} Request Failed: ${status} due to ${errorMessage}, (Aborted). ${sourceMessage}`,
+      `${DESTINATION} Request Failed: "${status}" due to "${
+        errorMessage || JSON.stringify(response)
+      }", (Aborted) ${sourceMessage}`,
       400,
       destResponse,
     );
@@ -91,15 +97,17 @@ const getAccessToken = async (destination) => {
     const { httpResponse, processedResponse } = await handleHttpRequest('post', authUrl, {});
     // If the request fails, throwing error.
     if (!httpResponse.success) {
-      const { error } = httpResponse.response.response.data;
+      salesforceResponseHandler(processedResponse, `:- authentication failed during fetching access token.`, accessTokenKey);
+    }
+    const token = httpResponse.response.data;
+    // If the httpResponse.success is true it will not come, It's an extra security for developer's.
+    if (!token.access_token || !token.instance_url) {
       salesforceResponseHandler(
-        processedResponse.response,
-        `access token could not be generated due to ${error}`,
-        undefined,
+        processedResponse,
+        `:- authentication failed could not retrieve authorization token.`,
         accessTokenKey,
       );
     }
-    const token = httpResponse.response.data;
     return {
       token: `Bearer ${token.access_token}`,
       instanceUrl: token.instance_url,

diff --git a/src/v0/destinations/singular/transform.js b/src/v0/destinations/singular/transform.js
@@ -23,13 +23,15 @@ const responseBuilderSimple = (message, { Config }) => {
   const { eventAttributes, payload } = platformWisePayloadGenerator(message, sessionEvent);
   const endpoint = sessionEvent ? `${BASE_URL}/launch` : `${BASE_URL}/evt`;
 
-  if (!sessionEvent) {
-    const { products } = message.properties;
-    // If we have an event where we have an array of Products, example Order Completed
-    // We will convert the event to revenue events
-    if (products && Array.isArray(products)) {
-      return generateRevenuePayloadArray(products, payload, Config, eventAttributes);
-    }
+  // If we have an event where we have an array of Products, example Order Completed
+  // We will convert the event to revenue events
+  if (!sessionEvent && Array.isArray(message?.properties?.products)) {
+    return generateRevenuePayloadArray(
+      message.properties.products,
+      payload,
+      Config,
+      eventAttributes,
+    );
   }
 
   const response = {

diff --git a/src/v0/util/index.js b/src/v0/util/index.js
@@ -742,7 +742,10 @@ const handleMetadataForValue = (value, metadata, destKey, integrationsObj = null
         formattedVal = formattedVal.replace('https://', '').replace('http://', '');
         break;
       case 'domainUrlV2': {
-        const url = new URL(formattedVal);
+        const url = isValidUrl(formattedVal);
+        if (!url) {
+          throw new InstrumentationError(`Invalid URL: ${formattedVal}`);
+        }
         formattedVal = url.hostname.replace('www.', '');
         break;
       }
@@ -977,7 +980,7 @@ function getDestinationExternalID(message, type) {
   }
 
   if (Array.isArray(externalIdArray)) {
-    externalIdArray.forEach(extIdObj => {
+    externalIdArray.forEach((extIdObj) => {
       if (extIdObj.type === type) {
         destinationExternalId = extIdObj.id;
       }
@@ -1666,8 +1669,10 @@ const validatePhoneWithCountryCode = (phone) => {
  * @returns
  */
 const isHybridModeEnabled = (Config) => {
-  const { useNativeSDK, useNativeSDKToSend } = Config;
-  return useNativeSDK && !useNativeSDKToSend;
+  if (isDefinedAndNotNull(Config.useNativeSDK) && isDefinedAndNotNull(Config.useNativeSDKToSend)) {
+    return Config.useNativeSDK && !Config.useNativeSDKToSend;
+  }
+  return false;
 };
 
 /**

diff --git a/src/versionedRouter.js b/src/versionedRouter.js
@@ -138,6 +138,11 @@ async function compareWithCdkV2(destType, inputArr, feature, v0Result, v0Time) {
     const objectDiff = CommonUtils.objectDiff(v0Result, cdkResult);
     if (Object.keys(objectDiff).length > 0) {
       stats.counter('cdk_live_compare_test_failed', 1, { destType, feature });
+      logger.error(
+      `[LIVE_COMPARE_TEST] failed for destType=${destType}, feature=${feature}, diff_keys=${JSON.stringify(
+        Object.keys(objectDiff)
+      )}`
+      );
       // logger.error(
       //   `[LIVE_COMPARE_TEST] failed for destType=${destType}, feature=${feature}, diff=${JSON.stringify(
       //     objectDiff

diff --git a/test/__mocks__/data/salesforce/proxy_response.json b/test/__mocks__/data/salesforce/proxy_response.json
@@ -50,5 +50,23 @@
       ],
       "status": 503
     }
+  },
+  "https://rudderstack.my.salesforce.com/services/data/v50.0/sobjects/Lead/6": {
+    "response": {
+      "data": {
+        "error": "invalid_grant",
+        "error_description": "authentication failure"
+      },
+      "status": 400
+    }
+  },
+  "https://rudderstack.my.salesforce.com/services/data/v50.0/sobjects/Lead/7": {
+    "response": {
+      "data": {
+        "message": "Server Unavailable",
+        "errorCode": "SERVER_UNAVAILABLE"
+      },
+      "status": 503
+    }
   }
-}
+}
diff --git a/test/__tests__/__integration_util_test__/isHybridModeEnabled.test.js b/test/__tests__/__integration_util_test__/isHybridModeEnabled.test.js
@@ -0,0 +1,28 @@
+const { isHybridModeEnabled } = require("../../../src/v0/util");
+
+describe("isHybridModeEnabled util test", () => {
+    test("Both flags(useNativeSDK and useNativeSDKToSend) are not present", () => {
+      const isHybridMode = isHybridModeEnabled({});
+      expect(isHybridMode).toEqual(false);
+    });
+
+    test("useNativeSDKToSend flag is not present", () => {
+      const isHybridMode = isHybridModeEnabled({useNativeSDK: false});
+      expect(isHybridMode).toEqual(false);
+    });
+
+    test("Both flags(useNativeSDK and useNativeSDKToSend) are present and hybridMode is enabled", () => {
+      const isHybridMode = isHybridModeEnabled({useNativeSDK: true, useNativeSDKToSend: false});
+      expect(isHybridMode).toEqual(true);
+    });
+
+    test("Both flags(useNativeSDK and useNativeSDKToSend) are present and device mode is enabled", () => {
+      const isHybridMode = isHybridModeEnabled({useNativeSDK: true, useNativeSDKToSend: true});
+      expect(isHybridMode).toEqual(false);
+    });
+
+     test("Both flags(useNativeSDK and useNativeSDKToSend) are present and cloud mode is enabled", () => {
+      const isHybridMode = isHybridModeEnabled({useNativeSDK: false, useNativeSDKToSend: false});
+      expect(isHybridMode).toEqual(false);
+    });
+});
diff --git a/test/__tests__/data/active_campaign_router_input.json b/test/__tests__/data/active_campaign_router_input.json
@@ -141,5 +141,86 @@
       },
       "sentAt": "2019-10-14T11:15:53.296Z"
     }
+  },
+  {
+    "message": {
+      "name": "home",
+      "type": "page",
+      "sentAt": "2023-01-10T22:31:10.954Z",
+      "userId": "",
+      "channel": "web",
+      "context": {
+        "os": {
+          "name": "",
+          "version": ""
+        },
+        "app": {
+          "name": "RudderLabs JavaScript SDK",
+          "build": "1.0.0",
+          "version": "2.20.0",
+          "namespace": "com.rudderlabs.javascript"
+        },
+        "page": {
+          "url": "url",
+          "path": "path",
+          "title": "title",
+          "search": "search",
+          "tab_url": "https://simple-tenet.github.io/rudderstack-sample-site/",
+          "referrer": "referrer",
+          "initial_referrer": "$direct",
+          "referring_domain": "",
+          "initial_referring_domain": ""
+        },
+        "locale": "en-US",
+        "screen": {
+          "width": 1512,
+          "height": 982,
+          "density": 2,
+          "innerWidth": 846,
+          "innerHeight": 782
+        },
+        "traits": {},
+        "library": {
+          "name": "RudderLabs JavaScript SDK",
+          "version": "2.20.0"
+        },
+        "campaign": {},
+        "sessionId": 1673389635049,
+        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
+      },
+      "rudderId": "d37dd81f-5894-48ec-b312-09c590bab15c",
+      "messageId": "2774384870943800-0b15a357-c85d-4b22-916e-ed53286b742a",
+      "timestamp": "2023-01-10T22:31:11.601Z",
+      "properties": {
+        "url": "url",
+        "name": "home",
+        "path": "path",
+        "title": "title",
+        "search": "search",
+        "tab_url": "https://simple-tenet.github.io/rudderstack-sample-site/",
+        "referrer": "referrer",
+        "initial_referrer": "$direct",
+        "referring_domain": "",
+        "initial_referring_domain": ""
+      },
+      "receivedAt": "2023-01-10T22:31:11.612Z",
+      "request_ip": "0.0.0.20",
+      "anonymousId": "878e8f5f-9b6c-4aef-b5d3-1b970a13f17a",
+      "integrations": {
+        "All": true
+      },
+      "originalTimestamp": "2023-01-10T22:31:10.943Z"
+    },
+    "destination": {
+      "Config": {
+        "apiKey": "fbee74a147828e2932c701d19dc1f2dcfa4ac0048be3aa3a88d427090a59dc1c0fa002f1",
+        "apiUrl": "https://active.campaigns.rudder.com",
+        "actid": "476550467",
+        "eventKey": "f8a866fddc721350fdc2fbbd2e5c43a6dddaaa03"
+      }
+    },
+    "metadata": {
+      "jobId": 5
+    }
   }
 ]