diff --git a/package.json b/package.json index aa5497ffc2..e8502b035a 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ }, "devDependencies": { "@peculiar/webcrypto": "^1.2.3", + "@types/aws4": "^1.11.2", "@types/chance": "^1.1.3", "@types/cors": "^2.8.12", "@types/express": "^4.17.16", @@ -107,6 +108,7 @@ } }, "dependencies": { + "aws4": "^1.12.0", "chance": "^1.1.8" } } diff --git a/packages/destination-actions/src/destinations/index.ts b/packages/destination-actions/src/destinations/index.ts index 4ce0626eee..8939207425 100644 --- a/packages/destination-actions/src/destinations/index.ts +++ b/packages/destination-actions/src/destinations/index.ts @@ -98,6 +98,7 @@ register('643fdf094cfdbcf1bcccbc42', './usermaven') register('6440068936c4fb9f699b0645', './the-trade-desk-crm') register('6447ca8bfaa773a2ba0777a0', './tiktok-offline-conversions') register('645babd9362d97b777391325', './iterable') +register('644ad6c6c4a87a3290450602', './liveramp-audiences') function register(id: MetadataId, destinationPath: string) { // eslint-disable-next-line @typescript-eslint/no-var-requires diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/__tests__/__snapshots__/snapshot.test.ts.snap b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/__tests__/__snapshots__/snapshot.test.ts.snap new file mode 100644 index 0000000000..146a6f8734 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/__tests__/__snapshots__/snapshot.test.ts.snap @@ -0,0 +1,47 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Testing snapshot for LiverampAudiences's audienceEntered destination action: all fields 1`] = ` +"audience_key,testType +\\"PywYAY\\",\\"PywYAY\\"" +`; + +exports[`Testing snapshot for LiverampAudiences's audienceEntered destination action: enquotated indentifier data 1`] = ` +Array [ + "\\"LCD TV,50\\"\\"\\"", + "\\"\\"\\"early-bird\\"\\" special\\"", + "\\"5'8\\"\\"\\"", +] +`; + +exports[`Testing snapshot for LiverampAudiences's audienceEntered destination action: required fields 1`] = ` +"audience_key,testType +\\"PywYAY\\",\\"PywYAY\\"" +`; + +exports[`Testing snapshot for LiverampAudiences's audienceEntered destination action: required fields 2`] = ` +Headers { + Symbol(map): Object { + "authorization": Array [ + "AWS4-HMAC-SHA256 Credential=PywYAY/19700101/PywYAY/s3/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date, Signature=3273c678e9edaf86b444eeadfd6021f8814141ee94adcb7409c7931c1f286dff", + ], + "content-length": Array [ + "39", + ], + "content-type": Array [ + "application/x-www-form-urlencoded; charset=utf-8", + ], + "host": Array [ + "PywYAY.s3.amazonaws.com", + ], + "user-agent": Array [ + "Segment (Actions)", + ], + "x-amz-content-sha256": Array [ + "c52b8202716894946461f92ea7d3ae902483d4c0339c06e616618d84bce9d642", + ], + "x-amz-date": Array [ + "19700101T000012Z", + ], + }, +} +`; diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/__tests__/snapshot.test.ts b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/__tests__/snapshot.test.ts new file mode 100644 index 0000000000..01a8be45c0 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/__tests__/snapshot.test.ts @@ -0,0 +1,91 @@ +import { createTestEvent, createTestIntegration } from '@segment/actions-core' +import { generateTestData } from '../../../../lib/test-data' +import destination from '../../index' +import nock from 'nock' +import { enquoteIdentifier } from '../operations' + +const testDestination = createTestIntegration(destination) +const actionSlug = 'audienceEntered' +const destinationSlug = 'LiverampAudiences' +const seedName = `${destinationSlug}#${actionSlug}` + +describe(`Testing snapshot for ${destinationSlug}'s ${actionSlug} destination action:`, () => { + beforeAll(() => { + const mockDate = new Date(12345) + jest.spyOn(global, 'Date').mockImplementation(() => mockDate as unknown as string) + }) + + it('required fields', async () => { + const action = destination.actions[actionSlug] + const [eventData, settingsData] = generateTestData(seedName, destination, action, true) + eventData.delimiter = ',' + settingsData.upload_mode = 'S3' + nock(/.*/).persist().get(/.*/).reply(200) + nock(/.*/).persist().post(/.*/).reply(200) + nock(/.*/).persist().put(/.*/).reply(200) + + const event = createTestEvent({ + properties: eventData + }) + + const responses = await testDestination.testAction(actionSlug, { + event: event, + mapping: event.properties, + settings: settingsData, + auth: undefined + }) + + const request = responses[0].request + const rawBody = await request.text() + + try { + const json = JSON.parse(rawBody) + expect(json).toMatchSnapshot() + return + } catch (err) { + expect(rawBody).toMatchSnapshot() + } + + expect(request.headers).toMatchSnapshot() + }) + + it('all fields', async () => { + const action = destination.actions[actionSlug] + const [eventData, settingsData] = generateTestData(seedName, destination, action, false) + eventData.delimiter = ',' + settingsData.upload_mode = 'S3' + + nock(/.*/).persist().get(/.*/).reply(200) + nock(/.*/).persist().post(/.*/).reply(200) + nock(/.*/).persist().put(/.*/).reply(200) + + const event = createTestEvent({ + properties: eventData + }) + + const responses = await testDestination.testAction(actionSlug, { + event: event, + mapping: event.properties, + settings: settingsData, + auth: undefined + }) + + const request = responses[0].request + const rawBody = await request.text() + + try { + const json = JSON.parse(rawBody) + expect(json).toMatchSnapshot() + return + } catch (err) { + expect(rawBody).toMatchSnapshot() + } + }) + + it('enquotated indentifier data', async () => { + const identifiers = [`LCD TV,50"`, `"early-bird" special`, `5'8"`] + const enquotedIdentifiers = identifiers.map(enquoteIdentifier) + + expect(enquotedIdentifiers).toMatchSnapshot() + }) +}) diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/generated-types.ts b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/generated-types.ts new file mode 100644 index 0000000000..26ff916f55 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/generated-types.ts @@ -0,0 +1,26 @@ +// Generated file. DO NOT MODIFY IT BY HAND. + +export interface Payload { + /** + * Identifies the user within the entered audience. + */ + audience_key: string + /** + * Additional data pertaining to the user. + */ + identifier_data?: { + [k: string]: unknown + } + /** + * Character used to separate tokens in the resulting file. + */ + delimiter: string + /** + * Name of the audience the user has entered. + */ + audience_name: string + /** + * Datetime at which the event was received. Used to disambiguate the resulting file. + */ + received_at: string | number +} diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/index.ts b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/index.ts new file mode 100644 index 0000000000..698e7a2188 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/index.ts @@ -0,0 +1,56 @@ +import { ActionDefinition } from '@segment/actions-core' +import type { Settings } from '../generated-types' +import type { Payload } from './generated-types' +import { processData } from './operations' + +const action: ActionDefinition = { + title: 'Audience Entered', + description: 'Uploads audience membership data to a file for LiveRamp ingestion.', + defaultSubscription: 'event = "Audience Entered"', + fields: { + audience_key: { + label: 'Audience Key', + description: 'Identifies the user within the entered audience.', + type: 'string', + required: true, + default: { '@path': '$.userId' } + }, + identifier_data: { + label: 'Identifier Data', + description: `Additional data pertaining to the user.`, + type: 'object', + required: false, + defaultObjectUI: 'keyvalue:only', + default: { '@path': '$.context.traits' } + }, + delimiter: { + label: 'Delimeter', + description: `Character used to separate tokens in the resulting file.`, + type: 'string', + required: true, + default: ',' + }, + audience_name: { + label: 'Audience name', + description: `Name of the audience the user has entered.`, + type: 'string', + required: true, + default: { '@path': '$.properties.audience_key' } + }, + received_at: { + label: 'Received At', + description: `Datetime at which the event was received. Used to disambiguate the resulting file.`, + type: 'datetime', + required: true, + default: { '@path': '$.receivedAt' } + } + }, + perform: async (request, { settings, payload }) => { + return processData(request, settings, [payload]) + }, + performBatch: (request, { settings, payload }) => { + return processData(request, settings, payload) + } +} + +export default action diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/operations.ts b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/operations.ts new file mode 100644 index 0000000000..099db4bce1 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/audienceEntered/operations.ts @@ -0,0 +1,59 @@ +import { PayloadValidationError, RequestClient } from '@segment/actions-core' +import type { Settings } from '../generated-types' +import type { Payload } from './generated-types' +import { uploadS3, validateS3 } from '../s3' + +async function processData(request: RequestClient, settings: Settings, payloads: Payload[]) { + // STRATCONN-2554: Add support for SFTP + if (settings.upload_mode == 'S3') { + validateS3(settings) + } else { + throw new PayloadValidationError(`Unrecognized upload mode: ${settings.upload_mode}`) + } + // STRATCONN-2584: error if less than 25 elements in payload + + // Prepare header row. Expected format: + // liveramp_audience_key[1],identifier_data[0..n] + const rows = [] + const headers = ['audience_key'] + if (payloads[0].identifier_data) { + for (const identifier of Object.getOwnPropertyNames(payloads[0].identifier_data)) { + headers.push(identifier) + } + } + rows.push(headers.join(payloads[0].delimiter)) + + // Prepare data rows + for (const payload of payloads) { + const row = [] + row.push(payload.audience_key) + if (payload.identifier_data) { + for (const identifier of Object.getOwnPropertyNames(payload.identifier_data)) { + row.push(payload.identifier_data[identifier] as string) + } + } + rows.push(row.map(enquoteIdentifier).join(payload.delimiter)) + } + + // STRATCONN-2584: verify multiple emails are handled + const filename = `${payloads[0].audience_name}_PII_${payloads[0].received_at}.csv` + const fileContent = rows.join('\n') + + if (settings.upload_mode == 'S3') { + return await uploadS3(settings, filename, fileContent, request) + } +} + +/* + To avoid collision with delimeters, we should surround identifiers with quotation marks. + https://docs.liveramp.com/connect/en/formatting-file-data.html#idm45998667347936 + + Examples: + LCD TV -> "LCD TV" + LCD TV,50" -> "LCD TV,50""" +*/ +function enquoteIdentifier(identifier: string) { + return `"${identifier.replace(/"/g, '""')}"` +} + +export { processData, enquoteIdentifier } diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/generated-types.ts b/packages/destination-actions/src/destinations/liveramp-audiences/generated-types.ts new file mode 100644 index 0000000000..198cdc54dc --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/generated-types.ts @@ -0,0 +1,36 @@ +// Generated file. DO NOT MODIFY IT BY HAND. + +export interface Settings { + /** + * Choose delivery route for the files + */ + upload_mode: string + /** + * IAM user credentials with write permissions to the S3 bucket. + */ + s3_aws_access_key?: string + /** + * IAM user credentials with write permissions to the S3 bucket. + */ + s3_aws_secret_key?: string + /** + * Name of the S3 bucket where the files will be uploaded to. + */ + s3_aws_bucket_name?: string + /** + * Region where the S3 bucket is hosted. + */ + s3_aws_region?: string + /** + * User credentials for establishing an SFTP connection with LiveRamp. + */ + sftp_username?: string + /** + * User credentials for establishing an SFTP connection with LiveRamp. + */ + sftp_password?: string + /** + * Path within the SFTP server to upload the files to. + */ + sftp_folder_path?: string +} diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/index.ts b/packages/destination-actions/src/destinations/liveramp-audiences/index.ts new file mode 100644 index 0000000000..24c917d1b9 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/index.ts @@ -0,0 +1,73 @@ +import type { DestinationDefinition } from '@segment/actions-core' +import type { Settings } from './generated-types' + +import audienceEntered from './audienceEntered' + +const destination: DestinationDefinition = { + name: 'Liveramp Audiences', + slug: 'actions-liveramp-audiences', + mode: 'cloud', + + authentication: { + scheme: 'custom', + fields: { + upload_mode: { + label: 'Upload Mode', + description: 'Choose delivery route for the files', + type: 'string', + required: true, + default: 'S3', + choices: [ + { value: 'S3', label: 'S3' }, + { value: 'SFTP', label: 'SFTP' } + ] + }, + s3_aws_access_key: { + label: 'AWS Access Key ID (S3 only)', + description: 'IAM user credentials with write permissions to the S3 bucket.', + type: 'string' + }, + s3_aws_secret_key: { + label: 'AWS Secret Access Key (S3 only)', + description: 'IAM user credentials with write permissions to the S3 bucket.', + type: 'password' + }, + s3_aws_bucket_name: { + label: 'AWS Bucket Name (S3 only)', + description: 'Name of the S3 bucket where the files will be uploaded to.', + type: 'string' + }, + s3_aws_region: { + label: 'AWS Region (S3 only)', + description: 'Region where the S3 bucket is hosted.', + type: 'string' + }, + sftp_username: { + label: 'Username (SFTP only)', + description: 'User credentials for establishing an SFTP connection with LiveRamp.', + type: 'string' + }, + sftp_password: { + label: 'Password (SFTP only)', + description: 'User credentials for establishing an SFTP connection with LiveRamp.', + type: 'password' + }, + sftp_folder_path: { + label: 'Folder Path (SFTP only)', + description: 'Path within the SFTP server to upload the files to.', + type: 'string' + } + }, + testAuthentication: (_) => { + // Return a request that tests/validates the user's credentials. + // If you do not have a way to validate the authentication fields safely, + // you can remove the `testAuthentication` function, though discouraged. + // TODO: Validate SFTP + } + }, + actions: { + audienceEntered + } +} + +export default destination diff --git a/packages/destination-actions/src/destinations/liveramp-audiences/s3.ts b/packages/destination-actions/src/destinations/liveramp-audiences/s3.ts new file mode 100644 index 0000000000..f43da74f45 --- /dev/null +++ b/packages/destination-actions/src/destinations/liveramp-audiences/s3.ts @@ -0,0 +1,50 @@ +import generateS3RequestOptions from '../../lib/AWS/s3' +import { InvalidAuthenticationError, ModifiedResponse, RequestOptions } from '@segment/actions-core' +import type { Settings } from './generated-types' + +function validateS3(settings: Settings) { + if (!settings.s3_aws_access_key) { + throw new InvalidAuthenticationError('Selected S3 upload mode, but missing AWS Access Key') + } + + if (!settings.s3_aws_secret_key) { + throw new InvalidAuthenticationError('Selected S3 upload mode, but missing AWS Secret Key') + } + + if (!settings.s3_aws_bucket_name) { + throw new InvalidAuthenticationError('Selected S3 upload mode, but missing AWS S3 bucket name') + } + + if (!settings.s3_aws_region) { + throw new InvalidAuthenticationError('Selected S3 upload mode, but missing AWS Region') + } +} + +async function uploadS3( + settings: Settings, + filename: string, + fileContent: string, + request: (url: string, options?: RequestOptions) => Promise> +) { + const method = 'PUT' + const opts = await generateS3RequestOptions( + settings.s3_aws_bucket_name as string, + settings.s3_aws_region as string, + filename, + method, + fileContent, + settings.s3_aws_access_key as string, + settings.s3_aws_secret_key as string + ) + if (!opts.headers || !opts.method || !opts.host || !opts.path) { + throw new InvalidAuthenticationError('Unable to generate signature header for AWS S3 request.') + } + + return await request(`https://${opts.host}/${opts.path}`, { + headers: opts.headers as Record, + method, + body: opts.body + }) +} + +export { validateS3, uploadS3 } diff --git a/packages/destination-actions/src/lib/AWS/s3.ts b/packages/destination-actions/src/lib/AWS/s3.ts new file mode 100644 index 0000000000..ab66cf2b1f --- /dev/null +++ b/packages/destination-actions/src/lib/AWS/s3.ts @@ -0,0 +1,25 @@ +import aws4 from 'aws4' + +async function generateS3RequestOptions( + bucketName: string, + region: string, + path: string, + method: string, + body: string | Buffer, + accessKeyId: string, + secretAccessKey: string +) { + const opts = { + host: `${bucketName}.s3.amazonaws.com`, + path, + body, + method, + region + } + return aws4.sign(opts, { + accessKeyId, + secretAccessKey + }) +} + +export default generateS3RequestOptions diff --git a/yarn.lock b/yarn.lock index 0431a0d625..3906b63007 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3738,6 +3738,13 @@ resolved "https://registry.yarnpkg.com/@types/asn1js/-/asn1js-2.0.2.tgz#bb1992291381b5f06e22a829f2ae009267cdf8c5" integrity sha512-t4YHCgtD+ERvH0FyxvNlYwJ2ezhqw7t+Ygh4urQ7dJER8i185JPv6oIM3ey5YQmGN6Zp9EMbpohkjZi9t3UxwA== +"@types/aws4@^1.11.2": + version "1.11.2" + resolved "https://registry.yarnpkg.com/@types/aws4/-/aws4-1.11.2.tgz#7700aabe4646f8868b5d2b20820d9583225e7b78" + integrity sha512-x0f96eBPrCCJzJxdPbUvDFRva4yPpINJzTuXXpmS2j9qLUpF2nyGzvXPlRziuGbCsPukwY4JfuO+8xwsoZLzGw== + dependencies: + "@types/node" "*" + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": version "7.1.16" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.16.tgz#bc12c74b7d65e82d29876b5d0baf5c625ac58702" @@ -5133,6 +5140,11 @@ aws4@^1.11.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== +aws4@^1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.12.0.tgz#ce1c9d143389679e253b314241ea9aa5cec980d3" + integrity sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg== + axios@^1.0.0: version "1.1.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"