fix: user deletion handlers implementation across destinations (#1748)

* fix: make all the destination user deletion handlers uniform * fix: update the metadata fetch code to send to bugsnag * fix: req metadata function in user deletion handler * fix: correction for the MAX_BATCH_SIZE value (#1720) Co-authored-by: Anant <[email protected]> * fix: update to add robust error handling (#1719) * fix: update deletion flows * fix: update test cases * fix: update test cases to remove the ones which require mocking * fix: handled specific success sattus for braze and intercom * fix: make config error for af as retryable * fix: user deletion handlers test cases (#1723) * feat: mock cases for clevertap * feat: mock cases for engage Co-authored-by: Anant <[email protected]> * fix(destination): avoid creating parallel delete user requests in clevertap (#1751) * fix: user deletion module update (#1750) * chore: coverage and testing for mixpanel and engage * fix: updated api and test coverage for intercom * refactor: coverage and testing for sendgrid * feat: Batch Support with testing and mocking for Amplitude * feat: Batch Support with testing and mocking for Braze * fix: conflicts * fix: conflicts+1 * fix: conflicts and resolved comments * fix: added comments * fix: resolved Comments * fix: resolved Conflicts * fix: resolved Comments * fix: resolved Comments+1 * fix: resolved Comments+2 Co-authored-by: Anant <[email protected]> * refactor: af user deletion (#1735) * refactor: af user deletion * fix: conflicts Co-authored-by: Anant <[email protected]> Co-authored-by: Utsab Chowdhury <[email protected]> Co-authored-by: Anant Jain <[email protected]> Co-authored-by: Anant <[email protected]>
rudderlabs · Jan 5, 2023 · 786cfe0 · 786cfe0
1 parent 4e98299
commit 786cfe0
Show file tree

Hide file tree

Showing 41 changed files with 12,920 additions and 599 deletions.
diff --git a/src/v0/destinations/af/deleteUsers.js b/src/v0/destinations/af/deleteUsers.js
@@ -1,12 +1,17 @@
 /* eslint-disable no-await-in-loop */
 /* eslint-disable no-param-reassign */
 const { httpPOST } = require("../../../adapters/network");
-const { generateUUID } = require("../../util");
+const {
+  processAxiosResponse,
+  getDynamicErrorType
+} = require("../../../adapters/utils/networkUtils");
+const { generateUUID, isHttpStatusSuccess } = require("../../util");
 const {
   ConfigurationError,
   InstrumentationError,
-  RetryableError
+  NetworkError
 } = require("../../util/errorTypes");
+const tags = require("../../util/tags");
 const { executeCommonValidations } = require("../../util/regulation-api");
 
 /**
@@ -23,12 +28,25 @@ const deleteUser = async (endpoint, body, identityType, identityValue) => {
   body.subject_identities[0].identity_type = identityType;
   body.subject_identities[0].identity_value = identityValue;
   const response = await httpPOST(endpoint, body);
-  return response;
+  const handledDelResponse = processAxiosResponse(response);
+  if (!isHttpStatusSuccess(handledDelResponse.status)) {
+    throw new NetworkError(
+      "User deletion request failed",
+      handledDelResponse.status,
+      {
+        [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(
+          handledDelResponse.status
+        )
+      },
+      handledDelResponse
+    );
+  }
+  return handledDelResponse;
 };
 
 /**
  * This function will help to delete the users one by one from the userAttributes array.
- * @param {*} userAttributes Array of objects with userId, emaail and phone
+ * @param {*} userAttributes Array of objects with userId, email and phone
  * @param {*} config Destination.Config provided in dashboard
  * @returns
  */
@@ -53,74 +71,61 @@ const userDeletionHandler = async (userAttributes, config) => {
       }
     );
     if (filteredStatusCallbackUrlsArray.length > 3) {
-      throw new ConfigurationError("you can send atmost 3 callBackUrls");
+      throw new ConfigurationError("You can send utmost 3 callBackUrls");
     }
     body.status_callback_urls = filteredStatusCallbackUrlsArray;
   }
   const endpoint = `https://hq1.appsflyer.com/gdpr/opengdpr_requests?api_token=${config.apiToken}`;
-  for (let i = 0; i < userAttributes.length; i += 1) {
-    const userAttributeKeys = Object.keys(userAttributes[i]);
-
-    if (userAttributeKeys.includes("appsflyer_id")) {
-      body.property_id = config.androidAppId
-        ? config.androidAppId
-        : config.appleAppId;
-      const response = await deleteUser(
-        endpoint,
-        body,
-        "appsflyer_id",
-        userAttributes[i].appsflyer_id
-      );
-      if (!response || !response.response) {
-        throw new RetryableError("Could not get response");
+  await Promise.all(
+    userAttributes.map(async ua => {
+      if (
+        !ua.android_advertising_id &&
+        !ua.ios_advertising_id &&
+        !ua.appsflyer_id
+      ) {
+        throw new InstrumentationError(
+          "none of the possible identityTypes i.e.(ios_advertising_id, android_advertising_id, appsflyer_id) is provided for deletion"
+        );
       }
-    } else {
-      if (userAttributeKeys.includes("ios_advertising_id")) {
+      /**
+       * Building the request Body in the following priority:
+       * appsflyer_id, ios_advertising_id, android_advertising_id
+       */
+      if (ua?.appsflyer_id) {
+        body.property_id = config.androidAppId
+          ? config.androidAppId
+          : config.appleAppId;
+        await deleteUser(endpoint, body, "appsflyer_id", ua.appsflyer_id);
+      } else if (ua?.ios_advertising_id) {
         body.property_id = config.appleAppId;
         if (!body.property_id) {
           throw new ConfigurationError(
             "appleAppId is required for ios_advertising_id type identifier"
           );
         }
-        const response = await deleteUser(
+        await deleteUser(
           endpoint,
           body,
           "ios_advertising_id",
-          userAttributes[i].ios_advertising_id
+          ua.ios_advertising_id
         );
-        if (!response || !response.response) {
-          throw new RetryableError("Could not get response");
-        }
-      }
-      if (userAttributeKeys.includes("android_advertising_id")) {
+      } else {
         body.property_id = config.androidAppId;
         if (!body.property_id) {
           throw new ConfigurationError(
             "androidAppId is required for android_advertising_id type identifier"
           );
         }
-        const response = await deleteUser(
+        await deleteUser(
           endpoint,
           body,
           "android_advertising_id",
-          userAttributes[i].android_advertising_id
+          ua.android_advertising_id
         );
-        if (!response || !response.response) {
-          throw new RetryableError("Could not get response");
-        }
       }
-    }
+    })
+  );
 
-    if (
-      !userAttributes[i].android_advertising_id &&
-      !userAttributes[i].ios_advertising_id &&
-      !userAttributes[i].appsflyer_id
-    ) {
-      throw new InstrumentationError(
-        "none of the possible identityTypes i.e.(ios_advertising_id, android_advertising_id, appsflyer_id) is provided for deletion"
-      );
-    }
-  }
   return { statusCode: 200, status: "successful" };
 };
 

diff --git a/src/v0/destinations/am/config.js b/src/v0/destinations/am/config.js
@@ -118,6 +118,8 @@ const events = Object.keys(Event);
 events.forEach(event => {
   nameToEventMap[Event[event].name] = Event[event];
 });
+// Ref : https://www.docs.developers.amplitude.com/analytics/apis/user-privacy-api/#response
+const DELETE_MAX_BATCH_SIZE = 100;
 const DESTINATION = "amplitude";
 
 module.exports = {
@@ -128,5 +130,6 @@ module.exports = {
   ConfigCategory,
   mappingConfig,
   nameToEventMap,
+  DELETE_MAX_BATCH_SIZE,
   batchEventsWithUserIdLengthLowerThanFive
 };
diff --git a/src/v0/destinations/am/deleteUsers.js b/src/v0/destinations/am/deleteUsers.js
@@ -1,15 +1,15 @@
 const btoa = require("btoa");
-const { httpSend } = require("../../../adapters/network");
-const { getDynamicErrorType } = require("../../../adapters/utils/networkUtils");
-
+const { httpPOST } = require("../../../adapters/network");
+const tags = require("../../util/tags");
 const {
-  NetworkError,
-  RetryableError,
-  InstrumentationError,
-  ConfigurationError
-} = require("../../util/errorTypes");
+  processAxiosResponse,
+  getDynamicErrorType
+} = require("../../../adapters/utils/networkUtils");
+const { isHttpStatusSuccess } = require("../../util");
+const { ConfigurationError, NetworkError } = require("../../util/errorTypes");
 const { executeCommonValidations } = require("../../util/regulation-api");
-const tags = require("../../util/tags");
+const { DELETE_MAX_BATCH_SIZE } = require("./config");
+const { getUserIdBatches } = require("../../util/deleteUserUtils");
 
 const userDeletionHandler = async (userAttributes, config) => {
   if (!config) {
@@ -20,43 +20,39 @@ const userDeletionHandler = async (userAttributes, config) => {
     throw new ConfigurationError("api key/secret for deletion not present");
   }
 
-  for (let i = 0; i < userAttributes.length; i += 1) {
-    const uId = userAttributes[i].userId;
-    if (!uId) {
-      throw new InstrumentationError("User id for deletion not present");
-    }
-    const data = { user_ids: [uId], requester: "RudderStack" };
-    const requestOptions = {
-      method: "post",
-      url: "https://amplitude.com/api/2/deletions/users",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Basic ${btoa(`${apiKey}:${apiSecret}`)}`
-      },
-      data
-    };
-    const resp = await httpSend(requestOptions);
-    if (!resp || !resp.response) {
-      throw new RetryableError("Could not get response");
-    }
-    if (
-      resp &&
-      resp.response &&
-      resp.response?.response &&
-      resp.response?.response?.status !== 200 // am sends 400 for any bad request or even if user id is not found. The text is also "Bad Request" so not handling user not found case
-    ) {
-      throw new NetworkError(
-        resp.response?.response?.statusText || "Error while deleting user",
-        resp.response?.response?.status,
-        {
-          [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(
-            resp.response?.response?.status
-          )
-        },
-        resp
-      );
-    }
-  }
+  const headers = {
+    "Content-Type": "application/json",
+    Authorization: `Basic ${btoa(`${apiKey}:${apiSecret}`)}`
+  };
+  // Ref : https://www.docs.developers.amplitude.com/analytics/apis/user-privacy-api/#response
+  const batchEvents = getUserIdBatches(userAttributes, DELETE_MAX_BATCH_SIZE);
+  const url = "https://amplitude.com/api/2/deletions/users";
+  await Promise.all(
+    batchEvents.map(async batch => {
+      const data = {
+        user_ids: batch,
+        requester: "RudderStack",
+        ignore_invalid_id: "true"
+      };
+      const requestOptions = {
+        headers
+      };
+      const resp = await httpPOST(url, data, requestOptions);
+      const handledDelResponse = processAxiosResponse(resp);
+      if (!isHttpStatusSuccess(handledDelResponse.status)) {
+        throw new NetworkError(
+          "User deletion request failed",
+          handledDelResponse.status,
+          {
+            [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(
+              handledDelResponse.status
+            )
+          },
+          handledDelResponse
+        );
+      }
+    })
+  );
   return { statusCode: 200, status: "successful" };
 };
 const processDeleteUsers = async event => {

diff --git a/src/v0/destinations/braze/config.js b/src/v0/destinations/braze/config.js
@@ -29,7 +29,9 @@ const BRAZE_PARTNER_NAME = "RudderStack";
 // Ref: https://www.braze.com/docs/api/endpoints/user_data/post_user_track/
 const TRACK_BRAZE_MAX_REQ_COUNT = 75;
 const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50;
+// https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/
 
+const DEL_MAX_BATCH_SIZE = 50;
 const DESTINATION = "braze";
 
 module.exports = {
@@ -41,5 +43,6 @@ module.exports = {
   BRAZE_PARTNER_NAME,
   TRACK_BRAZE_MAX_REQ_COUNT,
   IDENTIFY_BRAZE_MAX_REQ_COUNT,
-  DESTINATION
+  DESTINATION,
+  DEL_MAX_BATCH_SIZE
 };