Add updateHandler to Actions DV360 (segmentio#1726)

* Add updateHandler to Actions DV360

* Fix tests

* update yarn.lock

packages/cli/src/lib/server.ts

@@ -301,6 +301,7 @@ function setupRoutes(def: DestinationDefinition | null): void {
           if (Array.isArray(eventParams.data)) {
             // If no mapping or default mapping is provided, default to using the first payload across all events.
             eventParams.mapping = mapping || eventParams.data[0] || {}
+            eventParams.audienceSettings = req.body.payload[0]?.context?.personas?.audience_settings || {}
             await action.executeBatch(eventParams)
           } else {
             await action.execute(eventParams)

packages/destination-actions/package.json

@@ -39,6 +39,9 @@
   },
   "dependencies": {
     "@amplitude/ua-parser-js": "^0.7.25",
+    "@bufbuild/buf": "^1.28.0",
+    "@bufbuild/protobuf": "^1.4.2",
+    "@bufbuild/protoc-gen-es": "^1.4.2",
     "@segment/a1-notation": "^2.1.4",
     "@segment/actions-core": "^3.88.0",
     "@segment/actions-shared": "^1.70.0",

packages/destination-actions/src/destinations/display-video-360/__tests__/index.test.ts

@@ -9,19 +9,22 @@ const testDestination = createTestIntegration(Destination)
 const advertiserCreateAudienceUrl = CREATE_AUDIENCE_URL.replace('advertiserID', advertiserId)
 const advertiserGetAudienceUrl = GET_AUDIENCE_URL.replace('advertiserID', advertiserId)
 const expectedExternalID = `products/DISPLAY_VIDEO_ADVERTISER/customers/${advertiserId}/userLists/8457147615`
+const accountType = 'DISPLAY_VIDEO_ADVERTISER'
 
 const createAudienceInput = {
   settings: {},
   audienceName: '',
   audienceSettings: {
-    advertiserId: advertiserId
+    advertiserId: advertiserId,
+    accountType: accountType
   }
 }
 
 const getAudienceInput = {
   settings: {},
   audienceSettings: {
-    advertiserId: advertiserId
+    advertiserId: advertiserId,
+    accountType: accountType
   },
   audienceName: audienceName,
   externalId: expectedExternalID

packages/destination-actions/src/destinations/display-video-360/__tests__/shared.test.ts

@@ -0,0 +1,259 @@
+import nock from 'nock'
+import {
+  buildHeaders,
+  assembleRawOps,
+  bulkUploaderResponseHandler,
+  createUpdateRequest,
+  sendUpdateRequest
+} from '../shared'
+import { AudienceSettings, Settings } from '../generated-types'
+import { UpdateHandlerPayload } from '../types'
+import { UpdateUsersDataResponse, ErrorCode, ErrorInfo } from '../proto/protofile'
+import { StatsContext, Response } from '@segment/actions-core'
+import createRequestClient from '../../../../../core/src/create-request-client'
+
+const oneMockPayload: UpdateHandlerPayload = {
+  external_audience_id: 'products/DISPLAY_VIDEO_ADVERTISER/customers/123/userLists/456',
+  google_gid: 'CAESEHIV8HXNp0pFdHgi2rElMfk',
+  mobile_advertising_id: '3b6e47b3-1437-4ba2-b3c9-446e4d0cd1e5',
+  anonymous_id: 'my-anon-id-42',
+  enable_batching: true
+}
+
+const mockRequestClient = createRequestClient()
+
+const manyMockPayloads: UpdateHandlerPayload[] = [
+  oneMockPayload,
+  {
+    external_audience_id: 'products/DISPLAY_VIDEO_ADVERTISER/customers/123/userLists/456',
+    anonymous_id: 'my-anon-id-43',
+    enable_batching: true
+  },
+  {
+    external_audience_id: 'products/DISPLAY_VIDEO_ADVERTISER/customers/123/userLists/456',
+    google_gid: 'XNp0pFdHgi2rElMfk',
+    enable_batching: true
+  }
+]
+
+const mockStatsClient = {
+  incr: jest.fn(),
+  observe: jest.fn(),
+  _name: jest.fn(),
+  _tags: jest.fn(),
+  histogram: jest.fn(),
+  set: jest.fn()
+}
+
+const mockStatsContext = {
+  statsClient: mockStatsClient,
+  tags: []
+} as StatsContext
+
+const getRandomError = () => {
+  // possible errors for this stage are BAD_DATA, BAD_COOKIE, BAD_ATTRIBUTE_ID, BAD_NETWORK_ID.
+  const random = Math.floor(Math.random() * 4)
+  switch (random) {
+    case 0:
+      return ErrorCode.BAD_DATA
+    case 1:
+      return ErrorCode.BAD_COOKIE
+    case 2:
+      return ErrorCode.BAD_ATTRIBUTE_ID
+    case 3:
+      return ErrorCode.BAD_NETWORK_ID
+  }
+}
+
+// Mock only the error code. The contents of the response are not important.
+const createMockResponse = (errorCode: ErrorCode, payload: UpdateHandlerPayload[]) => {
+  const responseHandler = new UpdateUsersDataResponse()
+  responseHandler.status = errorCode
+
+  if (errorCode === ErrorCode.PARTIAL_SUCCESS) {
+    // Making assumptions about IdType and UserId here because
+    // we are not currently testing their content therefore, it doesn't matter.
+
+    const errors = payload.map((p) => {
+      const errorInfo = new ErrorInfo()
+      errorInfo.errorCode = getRandomError()
+      errorInfo.userListId = BigInt(p.external_audience_id.split('/').pop() || '-1')
+      errorInfo.userIdType = 0
+      errorInfo.userId = p.google_gid || p.mobile_advertising_id || p.anonymous_id || ''
+      return errorInfo
+    })
+
+    responseHandler.errors = errors
+  }
+
+  const b = Buffer.from(responseHandler.toBinary())
+  const arrayBuffer = b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength)
+
+  return new Response(arrayBuffer, { status: errorCode === ErrorCode.NO_ERROR ? 200 : 400 })
+}
+
+describe('shared', () => {
+  describe('buildHeaders', () => {
+    it('should build headers correctly', () => {
+      const audienceSettings: AudienceSettings = {
+        advertiserId: '123',
+        accountType: 'DISPLAY_VIDEO_ADVERTISER'
+      }
+      const settings: Settings = {
+        oauth: {
+          accessToken: 'real-token'
+        }
+      }
+      const result = buildHeaders(audienceSettings, settings)
+      expect(result).toEqual({
+        Authorization: 'Bearer real-token',
+        'Content-Type': 'application/json',
+        'Login-Customer-Id': 'products/DISPLAY_VIDEO_ADVERTISER/customers/123'
+      })
+    })
+  })
+
+  describe('assembleRawOps', () => {
+    it('should return an array of UserOperation objects with IDFA', () => {
+      const results = assembleRawOps(oneMockPayload, 'add')
+      expect(results).toEqual([
+        {
+          UserId: 'CAESEHIV8HXNp0pFdHgi2rElMfk',
+          UserIdType: 0,
+          UserListId: 456,
+          Delete: false
+        },
+        {
+          UserId: '3b6e47b3-1437-4ba2-b3c9-446e4d0cd1e5',
+          UserIdType: 1,
+          UserListId: 456,
+          Delete: false
+        },
+        {
+          UserId: 'my-anon-id-42',
+          UserIdType: 4,
+          UserListId: 456,
+          Delete: false
+        }
+      ])
+    })
+
+    it('should return an array of UserOperation objects with Android Advertising ID', () => {
+      oneMockPayload.mobile_advertising_id = '3b6e47b314374ba2b3c9446e4d0cd1e5'
+
+      const results = assembleRawOps(oneMockPayload, 'remove')
+      expect(results).toEqual([
+        {
+          UserId: 'CAESEHIV8HXNp0pFdHgi2rElMfk',
+          UserIdType: 0,
+          UserListId: 456,
+          Delete: true
+        },
+        {
+          UserId: '3b6e47b314374ba2b3c9446e4d0cd1e5',
+          UserIdType: 2,
+          UserListId: 456,
+          Delete: true
+        },
+        {
+          UserId: 'my-anon-id-42',
+          UserIdType: 4,
+          UserListId: 456,
+          Delete: true
+        }
+      ])
+    })
+  })
+
+  // This method is used for both success and error cases.
+  // The easiest way to tell if something worked is to check the calls to statsClient
+  // The assumptions made around the payload are based on the error codes described in the protofile.
+  describe('bulkUploaderResponseHandler', () => {
+    it('handles success', async () => {
+      const mockResponse: Response = createMockResponse(ErrorCode.NO_ERROR, manyMockPayloads)
+      const statsName = 'addToAudience'
+
+      await bulkUploaderResponseHandler(mockResponse, statsName, mockStatsContext)
+      expect(mockStatsClient.incr).toHaveBeenCalledWith(`${statsName}.success`, 1, mockStatsContext.tags)
+    })
+
+    it('handles 400 error', async () => {
+      const mockResponse: Response = createMockResponse(ErrorCode.BAD_COOKIE, manyMockPayloads)
+      const statsName = 'addToAudience'
+
+      await bulkUploaderResponseHandler(mockResponse, statsName, mockStatsContext)
+      expect(mockStatsClient.incr).toHaveBeenCalledWith(`${statsName}.error.BAD_COOKIE`, 1, mockStatsContext.tags)
+    })
+
+    it('handles 500 error', async () => {
+      const mockResponse: Response = createMockResponse(ErrorCode.INTERNAL_ERROR, manyMockPayloads)
+      const statsName = 'removeFromAudience'
+
+      await expect(bulkUploaderResponseHandler(mockResponse, statsName, mockStatsContext)).rejects.toThrow(
+        'Bulk Uploader Internal Error'
+      )
+
+      expect(mockStatsClient.incr).toHaveBeenCalledWith(`${statsName}.error.INTERNAL_ERROR`, 1, mockStatsContext.tags)
+    })
+  })
+
+  // If the request is invalid, its serialization will throw an error.
+  // No need to test the contents of the object because that is covered in assembleRawOps.
+  describe('createUpdateRequest', () => {
+    it('should create an UpdateUsersDataRequest object with the correct number of operations', () => {
+      const r = createUpdateRequest(manyMockPayloads, 'add')
+      expect(r.ops.length).toEqual(5)
+    })
+
+    it('should throw an error when unable to create UpdateUsersDataRequest', () => {
+      const mockPayload = {
+        enable_batching: true
+      } as UpdateHandlerPayload
+      expect(() => createUpdateRequest([mockPayload], 'remove')).toThrowError()
+    })
+  })
+
+  // Not testing payload content here because it's covered by the bulkUploaderResponseHandler.
+  // Attempting to assemble a valid response payload is not worth the effort.
+  describe('sendUpdateRequest', () => {
+    it('should succeed', async () => {
+      nock('https://cm.g.doubleclick.net').post('/upload?nid=segment').reply(200)
+
+      const r = createUpdateRequest(manyMockPayloads, 'add')
+      await sendUpdateRequest(mockRequestClient, r, 'addToAudience', mockStatsContext)
+      expect(mockStatsClient.incr).toHaveBeenCalledWith('addToAudience.success', 1, mockStatsContext.tags)
+    })
+
+    // To gracefully fails means that the request was successful, but some of the operations failed.
+    // The response will contain a list of errors. Its content is unknown.
+    // The endpoint will return a 400 status code.
+    it('should gracefully fail', async () => {
+      nock('https://cm.g.doubleclick.net').post('/upload?nid=segment').reply(400)
+
+      UpdateUsersDataResponse.prototype.fromBinary = jest.fn(() => {
+        const responseHandler = new UpdateUsersDataResponse()
+        responseHandler.status = ErrorCode.PARTIAL_SUCCESS
+        responseHandler.errors = [
+          {
+            errorCode: ErrorCode.BAD_DATA,
+            userListId: BigInt(456),
+            userIdType: 0,
+            userId: 'CAESEHIV8HXNp0pFdHgi2rElMfk'
+          } as ErrorInfo
+        ]
+        return responseHandler
+      })
+
+      const r = createUpdateRequest(manyMockPayloads, 'add')
+      await sendUpdateRequest(mockRequestClient, r, 'addToAudience', mockStatsContext)
+      expect(mockStatsClient.incr).toHaveBeenCalledWith('addToAudience.error.PARTIAL_SUCCESS', 1, mockStatsContext.tags)
+    })
+
+    it('should abruptly fail', async () => {
+      nock('https://cm.g.doubleclick.net').post('/upload?nid=segment').reply(500)
+
+      const r = createUpdateRequest(manyMockPayloads, 'add')
+      await expect(sendUpdateRequest(mockRequestClient, r, 'addToAudience', mockStatsContext)).rejects.toThrow()
+    })
+  })
+})

packages/destination-actions/src/destinations/display-video-360/addToAudience/index.ts

@@ -1,13 +1,30 @@
 import type { ActionDefinition } from '@segment/actions-core'
-import type { Settings } from '../generated-types'
+
+import type { Settings, AudienceSettings } from '../generated-types'
 import type { Payload } from './generated-types'
+import { handleUpdate } from '../shared'
+
+import { enable_batching, external_audience_id, google_gid, mobile_advertising_id, anonymous_id } from '../properties'
 
-const action: ActionDefinition<Settings, Payload> = {
+const action: ActionDefinition<Settings, Payload, AudienceSettings> = {
   title: 'Add to Audience',
-  description: 'Add users into an audience',
-  fields: {},
-  perform: () => {
-    return
+  description: 'Add a user to a Display & Video 360 audience.',
+  fields: {
+    enable_batching: { ...enable_batching },
+    external_audience_id: { ...external_audience_id },
+    anonymous_id: { ...anonymous_id },
+    mobile_advertising_id: { ...mobile_advertising_id },
+    google_gid: { ...google_gid }
+  },
+  perform: async (request, { payload, statsContext }) => {
+    statsContext?.statsClient?.incr('addToAudience', 1, statsContext?.tags)
+    await handleUpdate(request, [payload], 'add', statsContext)
+    return { success: true }
+  },
+  performBatch: async (request, { payload, statsContext }) => {
+    statsContext?.statsClient?.incr('addToAudience.batch', 1, statsContext?.tags)
+    await handleUpdate(request, payload, 'add', statsContext)
+    return { success: true }
   }
 }
 

packages/destination-actions/src/destinations/display-video-360/constants.ts

@@ -1,4 +1,7 @@
 export const GOOGLE_API_VERSION = 'v2'
-export const BASE_URL = `https://audiencepartner.googleapis.com/${GOOGLE_API_VERSION}/products/DISPLAY_VIDEO_ADVERTISER/customers/advertiserID/`
+// accountType and advertiserID are used as markers to be replaced in the code. DO NOT REMOVE THEM.
+export const BASE_URL = `https://audiencepartner.googleapis.com/${GOOGLE_API_VERSION}/products/accountType/customers/advertiserID/`
 export const CREATE_AUDIENCE_URL = `${BASE_URL}userLists:mutate`
 export const GET_AUDIENCE_URL = `${BASE_URL}audiencePartner:searchStream`
+export const OAUTH_URL = 'https://accounts.google.com/o/oauth2/token'
+export const USER_UPLOAD_ENDPOINT = 'https://cm.g.doubleclick.net/upload?nid=segment'