diff --git a/__mocks__/data/marketo_static_list/proxy_response.json b/__mocks__/data/marketo_static_list/proxy_response.json new file mode 100644 index 00000000000..4be224e504c --- /dev/null +++ b/__mocks__/data/marketo_static_list/proxy_response.json @@ -0,0 +1,38 @@ +{ + "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3": { + "data": { + "requestId": "68d8#1846058ee27", + "success": false, + "errors": [ + { + "code": "601", + "message": "Access token invalid" + } + ] + }, + "status": 200 + }, + "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2": { + "data": { + "requestId": "12d3c#1846057dce2", + "result": [ + { + "id": 1, + "status": "added" + }, + { + "id": 2, + "status": "skipped", + "reasons": [ + { + "code": "1004", + "message": "Lead not found" + } + ] + } + ], + "success": true + }, + "status": 200 + } +} \ No newline at end of file diff --git a/__mocks__/network.js b/__mocks__/network.js index 5106e018208..6f46d4a150d 100644 --- a/__mocks__/network.js +++ b/__mocks__/network.js @@ -18,7 +18,9 @@ const urlDirectoryMap = { "graph.facebook.com": "facebook_pixel", "api.wootric.com": "wootric", "api.mautic.com": "mautic", - "adsapi.snapchat.com": "snapchat_custom_audience" + "adsapi.snapchat.com": "snapchat_custom_audience", + "marketo_acct_id_success.mktorest.com": "marketo_static_list" + }; function getData(url) { diff --git a/__tests__/data/marketo_output.json b/__tests__/data/marketo_output.json index 1dd07dcc936..09ccb48f436 100644 --- a/__tests__/data/marketo_output.json +++ b/__tests__/data/marketo_output.json @@ -122,7 +122,7 @@ }, { "statusCode": 500, - "error": "Request Failed for Marketo, Access Token Expired (Retryable).During fetching auth token" + "error": "Request Failed for marketo, Access Token Expired (Retryable).During fetching auth token" }, { "statusCode": 400, diff --git a/__tests__/data/marketo_proxy_output.json b/__tests__/data/marketo_proxy_output.json index 59c0a7b440c..731a7535450 100644 --- a/__tests__/data/marketo_proxy_output.json +++ b/__tests__/data/marketo_proxy_output.json @@ -21,7 +21,7 @@ { "output": { "status": 500, - "message": "Request Failed for Marketo, Access token invalid (Retryable).during Marketo Response Handling", + "message": "Request Failed for marketo, Access token invalid (Retryable).during Marketo Response Handling", "destinationResponse": { "response": { "requestId": "a61c#17daea5968a", @@ -47,7 +47,7 @@ { "output": { "status": 400, - "message": "Request Failed for Marketo, Requested resource not found (Aborted).during Marketo Response Handling", + "message": "Request Failed for marketo, Requested resource not found (Aborted).during Marketo Response Handling", "destinationResponse": { "response": { "requestId": "a61c#17daea5968a", @@ -73,7 +73,7 @@ { "output": { "status": 413, - "message": "[Marketo Response Handler] - Request failed with status: 413", + "message": "[marketo Response Handler] - Request failed with status: 413", "destinationResponse": { "response": { "code": 413, @@ -94,7 +94,7 @@ { "output": { "status": 500, - "message": "[Marketo Response Handler] - Request failed with status: 500", + "message": "[marketo Response Handler] - Request failed with status: 500", "destinationResponse": { "response": "", "status": 500 @@ -111,7 +111,7 @@ { "output": { "status": 500, - "message": "[Marketo Response Handler] - Request failed with status: 500", + "message": "[marketo Response Handler] - Request failed with status: 500", "destinationResponse": { "response": "", "status": 500 @@ -128,7 +128,7 @@ { "output": { "status": 400, - "message": "[Marketo Response Handler] - Request failed with status: 400", + "message": "[marketo Response Handler] - Request failed with status: 400", "destinationResponse": { "response": "[[ENOTFOUND] :: DNS lookup failed]", "status": 400 diff --git a/__tests__/data/marketo_router_output.json b/__tests__/data/marketo_router_output.json index 3da3b5da293..abe73fd0e0e 100644 --- a/__tests__/data/marketo_router_output.json +++ b/__tests__/data/marketo_router_output.json @@ -240,7 +240,7 @@ { "batched": false, "statusCode": 400, - "error": "Request Failed for Marketo, Lookup field 'userId' not found (Aborted).[Marketo Transformer]: During lead look up using email", + "error": "Request Failed for marketo, Lookup field 'userId' not found (Aborted).[Marketo Transformer]: During lead look up using email", "statTags": { "destType": "MARKETO", "stage": "transform", diff --git a/__tests__/data/marketo_static_list.json b/__tests__/data/marketo_static_list.json new file mode 100644 index 00000000000..f8bd11c0596 --- /dev/null +++ b/__tests__/data/marketo_static_list.json @@ -0,0 +1,2545 @@ +[ + { + "description": "adding and removing users and getting staticListId from externalId", + "input": { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 3421 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "remove": [ + { + "id": 4 + }, + { + "id": 5 + }, + { + "id": 6 + } + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + }, + "externalId": [ + { + "type": "marketoStaticListId", + "id": 1234 + } + ] + }, + "timestamp": "2020-02-02T00:23:09.544Z" + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "DELETE", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + }, + { + "description": "adding more than max limit users", + "input": { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + }, + { + "id": 4 + }, + { + "id": 5 + }, + { + "id": 6 + }, + { + "id": 7 + }, + { + "id": 8 + }, + { + "id": 9 + }, + { + "id": 10 + }, + { + "id": 11 + }, + { + "id": 12 + }, + { + "id": 13 + }, + { + "id": 14 + }, + { + "id": 15 + }, + { + "id": 16 + }, + { + "id": 17 + }, + { + "id": 18 + }, + { + "id": 19 + }, + { + "id": 20 + }, + { + "id": 21 + }, + { + "id": 22 + }, + { + "id": 23 + }, + { + "id": 24 + }, + { + "id": 25 + }, + { + "id": 26 + }, + { + "id": 27 + }, + { + "id": 28 + }, + { + "id": 29 + }, + { + "id": 30 + }, + { + "id": 31 + }, + { + "id": 32 + }, + { + "id": 33 + }, + { + "id": 34 + }, + { + "id": 35 + }, + { + "id": 36 + }, + { + "id": 37 + }, + { + "id": 38 + }, + { + "id": 39 + }, + { + "id": 40 + }, + { + "id": 41 + }, + { + "id": 42 + }, + { + "id": 43 + }, + { + "id": 44 + }, + { + "id": 45 + }, + { + "id": 46 + }, + { + "id": 47 + }, + { + "id": 48 + }, + { + "id": 49 + }, + { + "id": 50 + }, + { + "id": 51 + }, + { + "id": 52 + }, + { + "id": 53 + }, + { + "id": 54 + }, + { + "id": 55 + }, + { + "id": 56 + }, + { + "id": 57 + }, + { + "id": 58 + }, + { + "id": 59 + }, + { + "id": 60 + }, + { + "id": 61 + }, + { + "id": 62 + }, + { + "id": 63 + }, + { + "id": 64 + }, + { + "id": 65 + }, + { + "id": 66 + }, + { + "id": 67 + }, + { + "id": 68 + }, + { + "id": 69 + }, + { + "id": 70 + }, + { + "id": 71 + }, + { + "id": 72 + }, + { + "id": 73 + }, + { + "id": 74 + }, + { + "id": 75 + }, + { + "id": 76 + }, + { + "id": 77 + }, + { + "id": 78 + }, + { + "id": 79 + }, + { + "id": 80 + }, + { + "id": 81 + }, + { + "id": 82 + }, + { + "id": 83 + }, + { + "id": 84 + }, + { + "id": 85 + }, + { + "id": 86 + }, + { + "id": 87 + }, + { + "id": 88 + }, + { + "id": 89 + }, + { + "id": 90 + }, + { + "id": 91 + }, + { + "id": 92 + }, + { + "id": 93 + }, + { + "id": 94 + }, + { + "id": 95 + }, + { + "id": 96 + }, + { + "id": 97 + }, + { + "id": 98 + }, + { + "id": 99 + }, + { + "id": 100 + }, + { + "id": 101 + }, + { + "id": 102 + }, + { + "id": 103 + }, + { + "id": 104 + }, + { + "id": 105 + }, + { + "id": 106 + }, + { + "id": 107 + }, + { + "id": 108 + }, + { + "id": 109 + }, + { + "id": 110 + }, + { + "id": 111 + }, + { + "id": 112 + }, + { + "id": 113 + }, + { + "id": 114 + }, + { + "id": 115 + }, + { + "id": 116 + }, + { + "id": 117 + }, + { + "id": 118 + }, + { + "id": 119 + }, + { + "id": 120 + }, + { + "id": 121 + }, + { + "id": 122 + }, + { + "id": 123 + }, + { + "id": 124 + }, + { + "id": 125 + }, + { + "id": 126 + }, + { + "id": 127 + }, + { + "id": 128 + }, + { + "id": 129 + }, + { + "id": 130 + }, + { + "id": 131 + }, + { + "id": 132 + }, + { + "id": 133 + }, + { + "id": 134 + }, + { + "id": 135 + }, + { + "id": 136 + }, + { + "id": 137 + }, + { + "id": 138 + }, + { + "id": 139 + }, + { + "id": 140 + }, + { + "id": 141 + }, + { + "id": 142 + }, + { + "id": 143 + }, + { + "id": 144 + }, + { + "id": 145 + }, + { + "id": 146 + }, + { + "id": 147 + }, + { + "id": 148 + }, + { + "id": 149 + }, + { + "id": 150 + }, + { + "id": 151 + }, + { + "id": 152 + }, + { + "id": 153 + }, + { + "id": 154 + }, + { + "id": 155 + }, + { + "id": 156 + }, + { + "id": 157 + }, + { + "id": 158 + }, + { + "id": 159 + }, + { + "id": 160 + }, + { + "id": 161 + }, + { + "id": 162 + }, + { + "id": 163 + }, + { + "id": 164 + }, + { + "id": 165 + }, + { + "id": 166 + }, + { + "id": 167 + }, + { + "id": 168 + }, + { + "id": 169 + }, + { + "id": 170 + }, + { + "id": 171 + }, + { + "id": 172 + }, + { + "id": 173 + }, + { + "id": 174 + }, + { + "id": 175 + }, + { + "id": 176 + }, + { + "id": 177 + }, + { + "id": 178 + }, + { + "id": 179 + }, + { + "id": 180 + }, + { + "id": 181 + }, + { + "id": 182 + }, + { + "id": 183 + }, + { + "id": 184 + }, + { + "id": 185 + }, + { + "id": 186 + }, + { + "id": 187 + }, + { + "id": 188 + }, + { + "id": 189 + }, + { + "id": 190 + }, + { + "id": 191 + }, + { + "id": 192 + }, + { + "id": 193 + }, + { + "id": 194 + }, + { + "id": 195 + }, + { + "id": 196 + }, + { + "id": 197 + }, + { + "id": 198 + }, + { + "id": 199 + }, + { + "id": 200 + }, + { + "id": 201 + }, + { + "id": 202 + }, + { + "id": 203 + }, + { + "id": 204 + }, + { + "id": 205 + }, + { + "id": 206 + }, + { + "id": 207 + }, + { + "id": 208 + }, + { + "id": 209 + }, + { + "id": 210 + }, + { + "id": 211 + }, + { + "id": 212 + }, + { + "id": 213 + }, + { + "id": 214 + }, + { + "id": 215 + }, + { + "id": 216 + }, + { + "id": 217 + }, + { + "id": 218 + }, + { + "id": 219 + }, + { + "id": 220 + }, + { + "id": 221 + }, + { + "id": 222 + }, + { + "id": 223 + }, + { + "id": 224 + }, + { + "id": 225 + }, + { + "id": 226 + }, + { + "id": 227 + }, + { + "id": 228 + }, + { + "id": 229 + }, + { + "id": 230 + }, + { + "id": 231 + }, + { + "id": 232 + }, + { + "id": 233 + }, + { + "id": 234 + }, + { + "id": 235 + }, + { + "id": 236 + }, + { + "id": 237 + }, + { + "id": 238 + }, + { + "id": 239 + }, + { + "id": 240 + }, + { + "id": 241 + }, + { + "id": 242 + }, + { + "id": 243 + }, + { + "id": 244 + }, + { + "id": 245 + }, + { + "id": 246 + }, + { + "id": 247 + }, + { + "id": 248 + }, + { + "id": 249 + }, + { + "id": 250 + }, + { + "id": 251 + }, + { + "id": 252 + }, + { + "id": 253 + }, + { + "id": 254 + }, + { + "id": 255 + }, + { + "id": 256 + }, + { + "id": 257 + }, + { + "id": 258 + }, + { + "id": 259 + }, + { + "id": 260 + }, + { + "id": 261 + }, + { + "id": 262 + }, + { + "id": 263 + }, + { + "id": 264 + }, + { + "id": 265 + }, + { + "id": 266 + }, + { + "id": 267 + }, + { + "id": 268 + }, + { + "id": 269 + }, + { + "id": 270 + }, + { + "id": 271 + }, + { + "id": 272 + }, + { + "id": 273 + }, + { + "id": 274 + }, + { + "id": 275 + }, + { + "id": 276 + }, + { + "id": 277 + }, + { + "id": 278 + }, + { + "id": 279 + }, + { + "id": 280 + }, + { + "id": 281 + }, + { + "id": 282 + }, + { + "id": 283 + }, + { + "id": 284 + }, + { + "id": 285 + }, + { + "id": 286 + }, + { + "id": 287 + }, + { + "id": 288 + }, + { + "id": 289 + }, + { + "id": 290 + }, + { + "id": 291 + }, + { + "id": 292 + }, + { + "id": 293 + }, + { + "id": 294 + }, + { + "id": 295 + }, + { + "id": 296 + }, + { + "id": 297 + }, + { + "id": 298 + }, + { + "id": 299 + }, + { + "id": 300 + }, + { + "id": 301 + }, + { + "id": 302 + }, + { + "id": 303 + }, + { + "id": 304 + }, + { + "id": 305 + }, + { + "id": 306 + }, + { + "id": 307 + }, + { + "id": 308 + }, + { + "id": 309 + }, + { + "id": 310 + }, + { + "id": 311 + }, + { + "id": 312 + }, + { + "id": 313 + }, + { + "id": 314 + }, + { + "id": 315 + }, + { + "id": 316 + }, + { + "id": 317 + }, + { + "id": 318 + }, + { + "id": 319 + }, + { + "id": 320 + }, + { + "id": 321 + }, + { + "id": 322 + }, + { + "id": 323 + }, + { + "id": 324 + }, + { + "id": 325 + }, + { + "id": 326 + }, + { + "id": 327 + }, + { + "id": 328 + }, + { + "id": 329 + }, + { + "id": 330 + }, + { + "id": 331 + }, + { + "id": 332 + }, + { + "id": 333 + }, + { + "id": 334 + }, + { + "id": 335 + }, + { + "id": 336 + }, + { + "id": 337 + }, + { + "id": 338 + }, + { + "id": 339 + }, + { + "id": 340 + }, + { + "id": 341 + }, + { + "id": 342 + }, + { + "id": 343 + }, + { + "id": 344 + }, + { + "id": 345 + }, + { + "id": 346 + }, + { + "id": 347 + }, + { + "id": 348 + }, + { + "id": 349 + }, + { + "id": 350 + } + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + }, + { + "description": "removing more than max limit users", + "input": { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "remove": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + }, + { + "id": 4 + }, + { + "id": 5 + }, + { + "id": 6 + }, + { + "id": 7 + }, + { + "id": 8 + }, + { + "id": 9 + }, + { + "id": 10 + }, + { + "id": 11 + }, + { + "id": 12 + }, + { + "id": 13 + }, + { + "id": 14 + }, + { + "id": 15 + }, + { + "id": 16 + }, + { + "id": 17 + }, + { + "id": 18 + }, + { + "id": 19 + }, + { + "id": 20 + }, + { + "id": 21 + }, + { + "id": 22 + }, + { + "id": 23 + }, + { + "id": 24 + }, + { + "id": 25 + }, + { + "id": 26 + }, + { + "id": 27 + }, + { + "id": 28 + }, + { + "id": 29 + }, + { + "id": 30 + }, + { + "id": 31 + }, + { + "id": 32 + }, + { + "id": 33 + }, + { + "id": 34 + }, + { + "id": 35 + }, + { + "id": 36 + }, + { + "id": 37 + }, + { + "id": 38 + }, + { + "id": 39 + }, + { + "id": 40 + }, + { + "id": 41 + }, + { + "id": 42 + }, + { + "id": 43 + }, + { + "id": 44 + }, + { + "id": 45 + }, + { + "id": 46 + }, + { + "id": 47 + }, + { + "id": 48 + }, + { + "id": 49 + }, + { + "id": 50 + }, + { + "id": 51 + }, + { + "id": 52 + }, + { + "id": 53 + }, + { + "id": 54 + }, + { + "id": 55 + }, + { + "id": 56 + }, + { + "id": 57 + }, + { + "id": 58 + }, + { + "id": 59 + }, + { + "id": 60 + }, + { + "id": 61 + }, + { + "id": 62 + }, + { + "id": 63 + }, + { + "id": 64 + }, + { + "id": 65 + }, + { + "id": 66 + }, + { + "id": 67 + }, + { + "id": 68 + }, + { + "id": 69 + }, + { + "id": 70 + }, + { + "id": 71 + }, + { + "id": 72 + }, + { + "id": 73 + }, + { + "id": 74 + }, + { + "id": 75 + }, + { + "id": 76 + }, + { + "id": 77 + }, + { + "id": 78 + }, + { + "id": 79 + }, + { + "id": 80 + }, + { + "id": 81 + }, + { + "id": 82 + }, + { + "id": 83 + }, + { + "id": 84 + }, + { + "id": 85 + }, + { + "id": 86 + }, + { + "id": 87 + }, + { + "id": 88 + }, + { + "id": 89 + }, + { + "id": 90 + }, + { + "id": 91 + }, + { + "id": 92 + }, + { + "id": 93 + }, + { + "id": 94 + }, + { + "id": 95 + }, + { + "id": 96 + }, + { + "id": 97 + }, + { + "id": 98 + }, + { + "id": 99 + }, + { + "id": 100 + }, + { + "id": 101 + }, + { + "id": 102 + }, + { + "id": 103 + }, + { + "id": 104 + }, + { + "id": 105 + }, + { + "id": 106 + }, + { + "id": 107 + }, + { + "id": 108 + }, + { + "id": 109 + }, + { + "id": 110 + }, + { + "id": 111 + }, + { + "id": 112 + }, + { + "id": 113 + }, + { + "id": 114 + }, + { + "id": 115 + }, + { + "id": 116 + }, + { + "id": 117 + }, + { + "id": 118 + }, + { + "id": 119 + }, + { + "id": 120 + }, + { + "id": 121 + }, + { + "id": 122 + }, + { + "id": 123 + }, + { + "id": 124 + }, + { + "id": 125 + }, + { + "id": 126 + }, + { + "id": 127 + }, + { + "id": 128 + }, + { + "id": 129 + }, + { + "id": 130 + }, + { + "id": 131 + }, + { + "id": 132 + }, + { + "id": 133 + }, + { + "id": 134 + }, + { + "id": 135 + }, + { + "id": 136 + }, + { + "id": 137 + }, + { + "id": 138 + }, + { + "id": 139 + }, + { + "id": 140 + }, + { + "id": 141 + }, + { + "id": 142 + }, + { + "id": 143 + }, + { + "id": 144 + }, + { + "id": 145 + }, + { + "id": 146 + }, + { + "id": 147 + }, + { + "id": 148 + }, + { + "id": 149 + }, + { + "id": 150 + }, + { + "id": 151 + }, + { + "id": 152 + }, + { + "id": 153 + }, + { + "id": 154 + }, + { + "id": 155 + }, + { + "id": 156 + }, + { + "id": 157 + }, + { + "id": 158 + }, + { + "id": 159 + }, + { + "id": 160 + }, + { + "id": 161 + }, + { + "id": 162 + }, + { + "id": 163 + }, + { + "id": 164 + }, + { + "id": 165 + }, + { + "id": 166 + }, + { + "id": 167 + }, + { + "id": 168 + }, + { + "id": 169 + }, + { + "id": 170 + }, + { + "id": 171 + }, + { + "id": 172 + }, + { + "id": 173 + }, + { + "id": 174 + }, + { + "id": 175 + }, + { + "id": 176 + }, + { + "id": 177 + }, + { + "id": 178 + }, + { + "id": 179 + }, + { + "id": 180 + }, + { + "id": 181 + }, + { + "id": 182 + }, + { + "id": 183 + }, + { + "id": 184 + }, + { + "id": 185 + }, + { + "id": 186 + }, + { + "id": 187 + }, + { + "id": 188 + }, + { + "id": 189 + }, + { + "id": 190 + }, + { + "id": 191 + }, + { + "id": 192 + }, + { + "id": 193 + }, + { + "id": 194 + }, + { + "id": 195 + }, + { + "id": 196 + }, + { + "id": 197 + }, + { + "id": 198 + }, + { + "id": 199 + }, + { + "id": 200 + }, + { + "id": 201 + }, + { + "id": 202 + }, + { + "id": 203 + }, + { + "id": 204 + }, + { + "id": 205 + }, + { + "id": 206 + }, + { + "id": 207 + }, + { + "id": 208 + }, + { + "id": 209 + }, + { + "id": 210 + }, + { + "id": 211 + }, + { + "id": 212 + }, + { + "id": 213 + }, + { + "id": 214 + }, + { + "id": 215 + }, + { + "id": 216 + }, + { + "id": 217 + }, + { + "id": 218 + }, + { + "id": 219 + }, + { + "id": 220 + }, + { + "id": 221 + }, + { + "id": 222 + }, + { + "id": 223 + }, + { + "id": 224 + }, + { + "id": 225 + }, + { + "id": 226 + }, + { + "id": 227 + }, + { + "id": 228 + }, + { + "id": 229 + }, + { + "id": 230 + }, + { + "id": 231 + }, + { + "id": 232 + }, + { + "id": 233 + }, + { + "id": 234 + }, + { + "id": 235 + }, + { + "id": 236 + }, + { + "id": 237 + }, + { + "id": 238 + }, + { + "id": 239 + }, + { + "id": 240 + }, + { + "id": 241 + }, + { + "id": 242 + }, + { + "id": 243 + }, + { + "id": 244 + }, + { + "id": 245 + }, + { + "id": 246 + }, + { + "id": 247 + }, + { + "id": 248 + }, + { + "id": 249 + }, + { + "id": 250 + }, + { + "id": 251 + }, + { + "id": 252 + }, + { + "id": 253 + }, + { + "id": 254 + }, + { + "id": 255 + }, + { + "id": 256 + }, + { + "id": 257 + }, + { + "id": 258 + }, + { + "id": 259 + }, + { + "id": 260 + }, + { + "id": 261 + }, + { + "id": 262 + }, + { + "id": 263 + }, + { + "id": 264 + }, + { + "id": 265 + }, + { + "id": 266 + }, + { + "id": 267 + }, + { + "id": 268 + }, + { + "id": 269 + }, + { + "id": 270 + }, + { + "id": 271 + }, + { + "id": 272 + }, + { + "id": 273 + }, + { + "id": 274 + }, + { + "id": 275 + }, + { + "id": 276 + }, + { + "id": 277 + }, + { + "id": 278 + }, + { + "id": 279 + }, + { + "id": 280 + }, + { + "id": 281 + }, + { + "id": 282 + }, + { + "id": 283 + }, + { + "id": 284 + }, + { + "id": 285 + }, + { + "id": 286 + }, + { + "id": 287 + }, + { + "id": 288 + }, + { + "id": 289 + }, + { + "id": 290 + }, + { + "id": 291 + }, + { + "id": 292 + }, + { + "id": 293 + }, + { + "id": 294 + }, + { + "id": 295 + }, + { + "id": 296 + }, + { + "id": 297 + }, + { + "id": 298 + }, + { + "id": 299 + }, + { + "id": 300 + }, + { + "id": 301 + }, + { + "id": 302 + }, + { + "id": 303 + }, + { + "id": 304 + }, + { + "id": 305 + }, + { + "id": 306 + }, + { + "id": 307 + }, + { + "id": 308 + }, + { + "id": 309 + }, + { + "id": 310 + }, + { + "id": 311 + }, + { + "id": 312 + }, + { + "id": 313 + }, + { + "id": 314 + }, + { + "id": 315 + }, + { + "id": 316 + }, + { + "id": 317 + }, + { + "id": 318 + }, + { + "id": 319 + }, + { + "id": 320 + }, + { + "id": 321 + }, + { + "id": 322 + }, + { + "id": 323 + }, + { + "id": 324 + }, + { + "id": 325 + }, + { + "id": 326 + }, + { + "id": 327 + }, + { + "id": 328 + }, + { + "id": 329 + }, + { + "id": 330 + }, + { + "id": 331 + }, + { + "id": 332 + }, + { + "id": 333 + }, + { + "id": 334 + }, + { + "id": 335 + }, + { + "id": 336 + }, + { + "id": 337 + }, + { + "id": 338 + }, + { + "id": 339 + }, + { + "id": 340 + }, + { + "id": 341 + }, + { + "id": 342 + }, + { + "id": 343 + }, + { + "id": 344 + }, + { + "id": 345 + }, + { + "id": 346 + }, + { + "id": 347 + }, + { + "id": 348 + }, + { + "id": 349 + }, + { + "id": 350 + } + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + } + }, + "output": [ + { + "version": "1", + "type": "REST", + "method": "DELETE", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "DELETE", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ] + }, + { + "description": "unsupported message Type", + "input": { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "track", + "properties": { + "listData": { + "remove": [ + 1 + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + } + }, + "output": { + "error": "track call is not supported." + } + }, + { + "description": "Invalid leadIds format ", + "input": { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "remove": 1 + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + } + }, + "output": { + "error": "Invalid leadIds format or no leadIds found neither to add nor to remove." + } + }, + { + "description": "Only adding with remove not an array", + "input": { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + {"id":1} + ], + "remove":2 + + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + } + }, + "output": [{ + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }] + } +] \ No newline at end of file diff --git a/__tests__/data/marketo_static_list_proxy_input.json b/__tests__/data/marketo_static_list_proxy_input.json new file mode 100644 index 00000000000..b1cb2823590 --- /dev/null +++ b/__tests__/data/marketo_static_list_proxy_input.json @@ -0,0 +1,60 @@ +[ + { + "request": { + "body": { + "type": "REST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", + "method": "POST", + "userId": "", + "headers": { + "Authorization": "Bearer Incorrect_token", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "FORM": {}, + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {} + }, + "files": {} + } + } + }, + { + "request": { + "body": { + "type": "REST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2", + "method": "POST", + "userId": "", + "headers": { + "Authorization": "Bearer token", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "FORM": {}, + "JSON": { + "action": "createOrUpdate", + "input": [ + { + "City": "Tokyo", + "Country": "JP", + "Email": "gabi29@gmail.com", + "PostalCode": "100-0001", + "Title": "Owner", + "id": 1328328, + "userId": "gabi_userId_45" + } + ], + "lookupField": "id" + }, + "JSON_ARRAY": {}, + "XML": {} + }, + "files": {} + } + } + } +] \ No newline at end of file diff --git a/__tests__/data/marketo_static_list_proxy_output.json b/__tests__/data/marketo_static_list_proxy_output.json new file mode 100644 index 00000000000..e83df649300 --- /dev/null +++ b/__tests__/data/marketo_static_list_proxy_output.json @@ -0,0 +1,57 @@ +[ + { + "output": { + "status": 500, + "message": "Request Failed for Marketo Static List, Access token invalid (Retryable).during Marketo Static List Response Handling", + "destinationResponse": { + "response": { + "requestId": "68d8#1846058ee27", + "success": false, + "errors": [ + { + "code": "601", + "message": "Access token invalid" + } + ] + }, + "status": 200 + }, + "statTags": { + "destType": "MARKETO STATIC LIST", + "stage": "responseTransform", + "scope": "api", + "meta": "retryable", + "errorAt": "proxy" + } + } + }, + { + "output": { + "status": 200, + "message": "[Marketo Static List Response Handler] - Request Processed Successfully", + "destinationResponse": { + "response": { + "requestId": "12d3c#1846057dce2", + "success": true, + "result": [ + { + "id": 1, + "status": "added" + }, + { + "id": 2, + "status": "skipped", + "reasons": [ + { + "code": "1004", + "message": "Lead not found" + } + ] + } + ] + }, + "status": 200 + } + } + } +] \ No newline at end of file diff --git a/__tests__/data/marketo_static_list_router_input.json b/__tests__/data/marketo_static_list_router_input.json new file mode 100644 index 00000000000..5defa914ac6 --- /dev/null +++ b/__tests__/data/marketo_static_list_router_input.json @@ -0,0 +1,1211 @@ +[ + { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "remove": [ + { + "id": 4 + }, + { + "id": 5 + }, + { + "id": 6 + } + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 1 + } + }, + { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "audiencelist", + "properties": { + "listData": { + "add": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + }, + { + "id": 4 + }, + { + "id": 5 + }, + { + "id": 6 + }, + { + "id": 7 + }, + { + "id": 8 + }, + { + "id": 9 + }, + { + "id": 10 + }, + { + "id": 11 + }, + { + "id": 12 + }, + { + "id": 13 + }, + { + "id": 14 + }, + { + "id": 15 + }, + { + "id": 16 + }, + { + "id": 17 + }, + { + "id": 18 + }, + { + "id": 19 + }, + { + "id": 20 + }, + { + "id": 21 + }, + { + "id": 22 + }, + { + "id": 23 + }, + { + "id": 24 + }, + { + "id": 25 + }, + { + "id": 26 + }, + { + "id": 27 + }, + { + "id": 28 + }, + { + "id": 29 + }, + { + "id": 30 + }, + { + "id": 31 + }, + { + "id": 32 + }, + { + "id": 33 + }, + { + "id": 34 + }, + { + "id": 35 + }, + { + "id": 36 + }, + { + "id": 37 + }, + { + "id": 38 + }, + { + "id": 39 + }, + { + "id": 40 + }, + { + "id": 41 + }, + { + "id": 42 + }, + { + "id": 43 + }, + { + "id": 44 + }, + { + "id": 45 + }, + { + "id": 46 + }, + { + "id": 47 + }, + { + "id": 48 + }, + { + "id": 49 + }, + { + "id": 50 + }, + { + "id": 51 + }, + { + "id": 52 + }, + { + "id": 53 + }, + { + "id": 54 + }, + { + "id": 55 + }, + { + "id": 56 + }, + { + "id": 57 + }, + { + "id": 58 + }, + { + "id": 59 + }, + { + "id": 60 + }, + { + "id": 61 + }, + { + "id": 62 + }, + { + "id": 63 + }, + { + "id": 64 + }, + { + "id": 65 + }, + { + "id": 66 + }, + { + "id": 67 + }, + { + "id": 68 + }, + { + "id": 69 + }, + { + "id": 70 + }, + { + "id": 71 + }, + { + "id": 72 + }, + { + "id": 73 + }, + { + "id": 74 + }, + { + "id": 75 + }, + { + "id": 76 + }, + { + "id": 77 + }, + { + "id": 78 + }, + { + "id": 79 + }, + { + "id": 80 + }, + { + "id": 81 + }, + { + "id": 82 + }, + { + "id": 83 + }, + { + "id": 84 + }, + { + "id": 85 + }, + { + "id": 86 + }, + { + "id": 87 + }, + { + "id": 88 + }, + { + "id": 89 + }, + { + "id": 90 + }, + { + "id": 91 + }, + { + "id": 92 + }, + { + "id": 93 + }, + { + "id": 94 + }, + { + "id": 95 + }, + { + "id": 96 + }, + { + "id": 97 + }, + { + "id": 98 + }, + { + "id": 99 + }, + { + "id": 100 + }, + { + "id": 101 + }, + { + "id": 102 + }, + { + "id": 103 + }, + { + "id": 104 + }, + { + "id": 105 + }, + { + "id": 106 + }, + { + "id": 107 + }, + { + "id": 108 + }, + { + "id": 109 + }, + { + "id": 110 + }, + { + "id": 111 + }, + { + "id": 112 + }, + { + "id": 113 + }, + { + "id": 114 + }, + { + "id": 115 + }, + { + "id": 116 + }, + { + "id": 117 + }, + { + "id": 118 + }, + { + "id": 119 + }, + { + "id": 120 + }, + { + "id": 121 + }, + { + "id": 122 + }, + { + "id": 123 + }, + { + "id": 124 + }, + { + "id": 125 + }, + { + "id": 126 + }, + { + "id": 127 + }, + { + "id": 128 + }, + { + "id": 129 + }, + { + "id": 130 + }, + { + "id": 131 + }, + { + "id": 132 + }, + { + "id": 133 + }, + { + "id": 134 + }, + { + "id": 135 + }, + { + "id": 136 + }, + { + "id": 137 + }, + { + "id": 138 + }, + { + "id": 139 + }, + { + "id": 140 + }, + { + "id": 141 + }, + { + "id": 142 + }, + { + "id": 143 + }, + { + "id": 144 + }, + { + "id": 145 + }, + { + "id": 146 + }, + { + "id": 147 + }, + { + "id": 148 + }, + { + "id": 149 + }, + { + "id": 150 + }, + { + "id": 151 + }, + { + "id": 152 + }, + { + "id": 153 + }, + { + "id": 154 + }, + { + "id": 155 + }, + { + "id": 156 + }, + { + "id": 157 + }, + { + "id": 158 + }, + { + "id": 159 + }, + { + "id": 160 + }, + { + "id": 161 + }, + { + "id": 162 + }, + { + "id": 163 + }, + { + "id": 164 + }, + { + "id": 165 + }, + { + "id": 166 + }, + { + "id": 167 + }, + { + "id": 168 + }, + { + "id": 169 + }, + { + "id": 170 + }, + { + "id": 171 + }, + { + "id": 172 + }, + { + "id": 173 + }, + { + "id": 174 + }, + { + "id": 175 + }, + { + "id": 176 + }, + { + "id": 177 + }, + { + "id": 178 + }, + { + "id": 179 + }, + { + "id": 180 + }, + { + "id": 181 + }, + { + "id": 182 + }, + { + "id": 183 + }, + { + "id": 184 + }, + { + "id": 185 + }, + { + "id": 186 + }, + { + "id": 187 + }, + { + "id": 188 + }, + { + "id": 189 + }, + { + "id": 190 + }, + { + "id": 191 + }, + { + "id": 192 + }, + { + "id": 193 + }, + { + "id": 194 + }, + { + "id": 195 + }, + { + "id": 196 + }, + { + "id": 197 + }, + { + "id": 198 + }, + { + "id": 199 + }, + { + "id": 200 + }, + { + "id": 201 + }, + { + "id": 202 + }, + { + "id": 203 + }, + { + "id": 204 + }, + { + "id": 205 + }, + { + "id": 206 + }, + { + "id": 207 + }, + { + "id": 208 + }, + { + "id": 209 + }, + { + "id": 210 + }, + { + "id": 211 + }, + { + "id": 212 + }, + { + "id": 213 + }, + { + "id": 214 + }, + { + "id": 215 + }, + { + "id": 216 + }, + { + "id": 217 + }, + { + "id": 218 + }, + { + "id": 219 + }, + { + "id": 220 + }, + { + "id": 221 + }, + { + "id": 222 + }, + { + "id": 223 + }, + { + "id": 224 + }, + { + "id": 225 + }, + { + "id": 226 + }, + { + "id": 227 + }, + { + "id": 228 + }, + { + "id": 229 + }, + { + "id": 230 + }, + { + "id": 231 + }, + { + "id": 232 + }, + { + "id": 233 + }, + { + "id": 234 + }, + { + "id": 235 + }, + { + "id": 236 + }, + { + "id": 237 + }, + { + "id": 238 + }, + { + "id": 239 + }, + { + "id": 240 + }, + { + "id": 241 + }, + { + "id": 242 + }, + { + "id": 243 + }, + { + "id": 244 + }, + { + "id": 245 + }, + { + "id": 246 + }, + { + "id": 247 + }, + { + "id": 248 + }, + { + "id": 249 + }, + { + "id": 250 + }, + { + "id": 251 + }, + { + "id": 252 + }, + { + "id": 253 + }, + { + "id": 254 + }, + { + "id": 255 + }, + { + "id": 256 + }, + { + "id": 257 + }, + { + "id": 258 + }, + { + "id": 259 + }, + { + "id": 260 + }, + { + "id": 261 + }, + { + "id": 262 + }, + { + "id": 263 + }, + { + "id": 264 + }, + { + "id": 265 + }, + { + "id": 266 + }, + { + "id": 267 + }, + { + "id": 268 + }, + { + "id": 269 + }, + { + "id": 270 + }, + { + "id": 271 + }, + { + "id": 272 + }, + { + "id": 273 + }, + { + "id": 274 + }, + { + "id": 275 + }, + { + "id": 276 + }, + { + "id": 277 + }, + { + "id": 278 + }, + { + "id": 279 + }, + { + "id": 280 + }, + { + "id": 281 + }, + { + "id": 282 + }, + { + "id": 283 + }, + { + "id": 284 + }, + { + "id": 285 + }, + { + "id": 286 + }, + { + "id": 287 + }, + { + "id": 288 + }, + { + "id": 289 + }, + { + "id": 290 + }, + { + "id": 291 + }, + { + "id": 292 + }, + { + "id": 293 + }, + { + "id": 294 + }, + { + "id": 295 + }, + { + "id": 296 + }, + { + "id": 297 + }, + { + "id": 298 + }, + { + "id": 299 + }, + { + "id": 300 + }, + { + "id": 301 + }, + { + "id": 302 + }, + { + "id": 303 + }, + { + "id": 304 + }, + { + "id": 305 + }, + { + "id": 306 + }, + { + "id": 307 + }, + { + "id": 308 + }, + { + "id": 309 + }, + { + "id": 310 + }, + { + "id": 311 + }, + { + "id": 312 + }, + { + "id": 313 + }, + { + "id": 314 + }, + { + "id": 315 + }, + { + "id": 316 + }, + { + "id": 317 + }, + { + "id": 318 + }, + { + "id": 319 + }, + { + "id": 320 + }, + { + "id": 321 + }, + { + "id": 322 + }, + { + "id": 323 + }, + { + "id": 324 + }, + { + "id": 325 + }, + { + "id": 326 + }, + { + "id": 327 + }, + { + "id": 328 + }, + { + "id": 329 + }, + { + "id": 330 + }, + { + "id": 331 + }, + { + "id": 332 + }, + { + "id": 333 + }, + { + "id": 334 + }, + { + "id": 335 + }, + { + "id": 336 + }, + { + "id": 337 + }, + { + "id": 338 + }, + { + "id": 339 + }, + { + "id": 340 + }, + { + "id": 341 + }, + { + "id": 342 + }, + { + "id": 343 + }, + { + "id": 344 + }, + { + "id": 345 + }, + { + "id": 346 + }, + { + "id": 347 + }, + { + "id": 348 + }, + { + "id": 349 + }, + { + "id": 350 + } + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 2 + } + }, + { + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + }, + "message": { + "userId": "user 1", + "anonymousId": "anon-id-new", + "event": "event1", + "type": "track", + "properties": { + "listData": { + "remove": [ + 1 + ] + }, + "enablePartialFailure": true + }, + "context": { + "ip": "14.5.67.21", + "library": { + "name": "http" + } + }, + "timestamp": "2020-02-02T00:23:09.544Z" + }, + "metadata": { + "jobId": 3 + } + } +] \ No newline at end of file diff --git a/__tests__/data/marketo_static_list_router_metadata_input.json b/__tests__/data/marketo_static_list_router_metadata_input.json new file mode 100644 index 00000000000..51924e1c90f --- /dev/null +++ b/__tests__/data/marketo_static_list_router_metadata_input.json @@ -0,0 +1,84 @@ +{ + "batchedRequest": { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer access_token_success" + }, + "params": {}, + "body": { + "JSON": {}, + "XML": {}, + "JSON_ARRAY": {}, + "FORM": {} + }, + "files": {} + }, + "metadata": [ + { + "jobId": 1 + }, + { + "jobId": 2 + }, + { + "jobId": 3 + } + ], + "batched": false, + "statusCode": 200, + "destination": { + "Config": { + "accountId": "marketo_acct_id_success", + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "staticListId": 3027 + }, + "secretConfig": {}, + "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", + "name": "Marketo Static List", + "enabled": true, + "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmLd", + "deleted": false, + "createdAt": "2020-12-30T08:39:32.005Z", + "updatedAt": "2021-02-03T16:22:31.374Z", + "destinationDefinition": { + "config": { + "destConfig": { + "defaultConfig": [ + "accountId", + "clientId", + "clientSecret", + "staticListId" + ] + }, + "secretKeys": [ + "clientSecret" + ], + "excludeKeys": [], + "includeKeys": [], + "routerTransform": true, + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "reactnative" + ] + }, + "id": "1aIXqM806xAVm92nx07YwKbRrO9", + "name": "MARKETO_STATIC_LIST", + "displayName": "Marketo Static List", + "createdAt": "2020-04-09T09:24:31.794Z", + "updatedAt": "2021-01-11T11:03:28.103Z" + }, + "transformations": [], + "isConnectionEnabled": true, + "isProcessorEnabled": true + } +} \ No newline at end of file diff --git a/__tests__/data/marketo_static_list_router_metadata_output.json b/__tests__/data/marketo_static_list_router_metadata_output.json new file mode 100644 index 00000000000..6289ddb86b2 --- /dev/null +++ b/__tests__/data/marketo_static_list_router_metadata_output.json @@ -0,0 +1,20 @@ +[ + { + "destInfo": { + "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" + }, + "jobId": 1 + }, + { + "destInfo": { + "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" + }, + "jobId": 2 + }, + { + "destInfo": { + "authKey": "1mMy5cqbtfuaKZv1IhVQKnBdVwe" + }, + "jobId": 3 + } +] \ No newline at end of file diff --git a/__tests__/data/marketo_static_list_router_output.json b/__tests__/data/marketo_static_list_router_output.json new file mode 100644 index 00000000000..6149551529b --- /dev/null +++ b/__tests__/data/marketo_static_list_router_output.json @@ -0,0 +1,152 @@ +[ + { + "batchedRequest": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=1&id=2&id=3", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "DELETE", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=4&id=5&id=6", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ], + "metadata": [ + { + "jobId": 1 + } + ], + "batched": false, + "statusCode": 200, + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + } + }, + { + "batchedRequest": [ + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=0&id=1&id=2&id=3&id=4&id=5&id=6&id=7&id=8&id=9&id=10&id=11&id=12&id=13&id=14&id=15&id=16&id=17&id=18&id=19&id=20&id=21&id=22&id=23&id=24&id=25&id=26&id=27&id=28&id=29&id=30&id=31&id=32&id=33&id=34&id=35&id=36&id=37&id=38&id=39&id=40&id=41&id=42&id=43&id=44&id=45&id=46&id=47&id=48&id=49&id=50&id=51&id=52&id=53&id=54&id=55&id=56&id=57&id=58&id=59&id=60&id=61&id=62&id=63&id=64&id=65&id=66&id=67&id=68&id=69&id=70&id=71&id=72&id=73&id=74&id=75&id=76&id=77&id=78&id=79&id=80&id=81&id=82&id=83&id=84&id=85&id=86&id=87&id=88&id=89&id=90&id=91&id=92&id=93&id=94&id=95&id=96&id=97&id=98&id=99&id=100&id=101&id=102&id=103&id=104&id=105&id=106&id=107&id=108&id=109&id=110&id=111&id=112&id=113&id=114&id=115&id=116&id=117&id=118&id=119&id=120&id=121&id=122&id=123&id=124&id=125&id=126&id=127&id=128&id=129&id=130&id=131&id=132&id=133&id=134&id=135&id=136&id=137&id=138&id=139&id=140&id=141&id=142&id=143&id=144&id=145&id=146&id=147&id=148&id=149&id=150&id=151&id=152&id=153&id=154&id=155&id=156&id=157&id=158&id=159&id=160&id=161&id=162&id=163&id=164&id=165&id=166&id=167&id=168&id=169&id=170&id=171&id=172&id=173&id=174&id=175&id=176&id=177&id=178&id=179&id=180&id=181&id=182&id=183&id=184&id=185&id=186&id=187&id=188&id=189&id=190&id=191&id=192&id=193&id=194&id=195&id=196&id=197&id=198&id=199&id=200&id=201&id=202&id=203&id=204&id=205&id=206&id=207&id=208&id=209&id=210&id=211&id=212&id=213&id=214&id=215&id=216&id=217&id=218&id=219&id=220&id=221&id=222&id=223&id=224&id=225&id=226&id=227&id=228&id=229&id=230&id=231&id=232&id=233&id=234&id=235&id=236&id=237&id=238&id=239&id=240&id=241&id=242&id=243&id=244&id=245&id=246&id=247&id=248&id=249&id=250&id=251&id=252&id=253&id=254&id=255&id=256&id=257&id=258&id=259&id=260&id=261&id=262&id=263&id=264&id=265&id=266&id=267&id=268&id=269&id=270&id=271&id=272&id=273&id=274&id=275&id=276&id=277&id=278&id=279&id=280&id=281&id=282&id=283&id=284&id=285&id=286&id=287&id=288&id=289&id=290&id=291&id=292&id=293&id=294&id=295&id=296&id=297&id=298&id=299", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + }, + { + "version": "1", + "type": "REST", + "method": "POST", + "endpoint": "https://marketo_acct_id_success.mktorest.com/rest/v1/lists/1234/leads.json?id=300&id=301&id=302&id=303&id=304&id=305&id=306&id=307&id=308&id=309&id=310&id=311&id=312&id=313&id=314&id=315&id=316&id=317&id=318&id=319&id=320&id=321&id=322&id=323&id=324&id=325&id=326&id=327&id=328&id=329&id=330&id=331&id=332&id=333&id=334&id=335&id=336&id=337&id=338&id=339&id=340&id=341&id=342&id=343&id=344&id=345&id=346&id=347&id=348&id=349&id=350", + "headers": { + "Authorization": "Bearer access_token_success", + "Content-Type": "application/json" + }, + "params": {}, + "body": { + "JSON": {}, + "JSON_ARRAY": {}, + "XML": {}, + "FORM": {} + }, + "files": {} + } + ], + "metadata": [ + { + "jobId": 2 + } + ], + "batched": false, + "statusCode": 200, + "destination": { + "ID": "1zia9wKshXt80YksLmUdJnr7IHI", + "Name": "test_marketo", + "DestinationDefinition": { + "ID": "1iVQvTRMsPPyJzwol0ifH93QTQ6", + "Name": "MARKETO", + "DisplayName": "Marketo", + "transformAt": "processor", + "transformAtV1": "processor" + }, + "Config": { + "clientId": "marketo_client_id_success", + "clientSecret": "marketo_client_secret_success", + "accountId": "marketo_acct_id_success", + "staticListId": 1234 + }, + "Enabled": true, + "Transformations": [], + "IsProcessorEnabled": true + } + }, + { + "metadata": [ + { + "jobId": 3 + } + ], + "batched": false, + "statusCode": 400, + "error": "track call is not supported.", + "statTags": { + "destType": "MARKETO STATIC LIST", + "meta": "badEvent", + "scope": "transformation", + "stage": "transform" + } + } +] \ No newline at end of file diff --git a/__tests__/marketo_static_list.test.js b/__tests__/marketo_static_list.test.js new file mode 100644 index 00000000000..0c4969321bf --- /dev/null +++ b/__tests__/marketo_static_list.test.js @@ -0,0 +1,64 @@ +const integration = "marketo_static_list"; +const name = "marketo_static_list"; +const version = "v0"; + +const fs = require("fs"); +const path = require("path"); + +const transformer = require(`../${version}/destinations/${integration}/transform`); + +// Processor Test Data +const testDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}.json`) +); +const testData = JSON.parse(testDataFile); + +// Router Test Data +const inputRouterDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_input.json`) +); +const outputRouterDataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_output.json`) +); +const inputRouterData = JSON.parse(inputRouterDataFile); +const expectedRouterData = JSON.parse(outputRouterDataFile); + +// Router Metadata Test files +const inputRouterMetadataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_metadata_input.json`) +); +const outputRouterMetadataFile = fs.readFileSync( + path.resolve(__dirname, `./data/${integration}_router_metadata_output.json`) +); +const inputRouterMetadata = JSON.parse(inputRouterMetadataFile); +const expectedRouterMetadata = JSON.parse(outputRouterMetadataFile); + +describe(`${name} Tests`, () => { + describe("Processor", () => { + testData.forEach((dataPoint, index) => { + it(`${index}. ${integration} - ${dataPoint.description}`, async () => { + try { + const output = await transformer.process(dataPoint.input); + expect(output).toEqual(dataPoint.output); + } catch (error) { + expect(error.message).toEqual(dataPoint.output.error); + } + }); + }); + }); + + describe("Router Tests", () => { + it("Payload", async () => { + const routerOutput = await transformer.processRouterDest(inputRouterData); + expect(routerOutput).toEqual(expectedRouterData); + }); + }); + describe("Router Metadata Tests", () => { + it("Payload", async () => { + const routerMetadataOutput = await transformer.processMetadataForRouter( + inputRouterMetadata + ); + expect(routerMetadataOutput).toEqual(expectedRouterMetadata); + }); + }); +}); diff --git a/__tests__/proxy.test.js b/__tests__/proxy.test.js index 085ba73f29a..4ab5eb4f676 100644 --- a/__tests__/proxy.test.js +++ b/__tests__/proxy.test.js @@ -9,7 +9,8 @@ const destinations = [ "google_adwords_remarketing_lists", "google_adwords_enhanced_conversions", "facebook_pixel", - "snapchat_custom_audience" + "snapchat_custom_audience", + "marketo_static_list" ]; const service = require("../versionedRouter").handleProxyRequest; diff --git a/features.json b/features.json index 818b3a2dcf7..e48c768ab6d 100644 --- a/features.json +++ b/features.json @@ -44,6 +44,7 @@ "FACEBOOK_OFFLINE_CONVERSIONS": true, "MAILJET": true, "SNAPCHAT_CUSTOM_AUDIENCE": true, + "MARKETO_STATIC_LIST": true, "CAMPAIGN_MANAGER": true, "SENDGRID": true } diff --git a/v0/destinations/marketo/networkHandler.js b/v0/destinations/marketo/networkHandler.js index 1d3c2d47698..c6e7ba643ca 100644 --- a/v0/destinations/marketo/networkHandler.js +++ b/v0/destinations/marketo/networkHandler.js @@ -20,7 +20,8 @@ const responseHandler = (destinationResponse, destType) => { "during Marketo Response Handling", TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM, destinationResponse?.rudderJobMetadata, - authCache + authCache, + "marketo" ); // else successfully return status, message and original destination response return { diff --git a/v0/destinations/marketo/transform.js b/v0/destinations/marketo/transform.js index 77ceae2e453..c45542471a3 100644 --- a/v0/destinations/marketo/transform.js +++ b/v0/destinations/marketo/transform.js @@ -650,6 +650,7 @@ function processMetadataForRouter(output) { const { metadata, destination } = output; const clonedMetadata = cloneDeep(metadata); clonedMetadata.forEach(metadataElement => { + // eslint-disable-next-line no-param-reassign metadataElement.destInfo = { authKey: destination?.ID }; }); return clonedMetadata; @@ -659,5 +660,6 @@ module.exports = { process, processRouterDest, processMetadataForRouter, - authCache + authCache, + getAuthToken }; diff --git a/v0/destinations/marketo/util.js b/v0/destinations/marketo/util.js index 58ac462750d..1287cc90623 100644 --- a/v0/destinations/marketo/util.js +++ b/v0/destinations/marketo/util.js @@ -20,7 +20,10 @@ const MARKETO_ABORTABLE_CODES = [ "609", "610", "612", - "1006" + "1006", + "1013", + "1004", + "1001" ]; const MARKETO_THROTTLED_CODES = ["502", "606", "607", "608", "615"]; const { DESTINATION } = require("./config"); @@ -30,13 +33,13 @@ const logger = require("../../../logger"); const marketoApplicationErrorHandler = ( marketoResponse, sourceMessage, - stage + destination ) => { const { response } = marketoResponse; const { errors } = response; if (errors && MARKETO_ABORTABLE_CODES.indexOf(errors[0].code) > -1) { throw new ApiError( - `Request Failed for Marketo, ${errors[0].message} (Aborted).${sourceMessage}`, + `Request Failed for ${destination}, ${errors[0].message} (Aborted).${sourceMessage}`, 400, { scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, @@ -44,11 +47,11 @@ const marketoApplicationErrorHandler = ( }, marketoResponse, undefined, // represents authErrorCategory - DESTINATION + destination ); } else if (errors && MARKETO_THROTTLED_CODES.indexOf(errors[0].code) > -1) { throw new ApiError( - `Request Failed for Marketo, ${errors[0].message} (Throttled).${sourceMessage}`, + `Request Failed for ${destination}, ${errors[0].message} (Throttled).${sourceMessage}`, 429, { scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, @@ -56,11 +59,11 @@ const marketoApplicationErrorHandler = ( }, marketoResponse, undefined, - DESTINATION + destination ); } else if (errors && MARKETO_RETRYABLE_CODES.indexOf(errors[0].code) > -1) { throw new ApiError( - `Request Failed for Marketo, ${errors[0].message} (Retryable).${sourceMessage}`, + `Request Failed for ${destination}, ${errors[0].message} (Retryable).${sourceMessage}`, 500, { scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, @@ -68,7 +71,7 @@ const marketoApplicationErrorHandler = ( }, marketoResponse, undefined, - DESTINATION + destination ); } }; @@ -78,13 +81,14 @@ const marketoResponseHandler = ( sourceMessage, stage, rudderJobMetadata, - authCache + authCache, + destination = DESTINATION ) => { const { status, response } = destResponse; // if the responsee from destination is not a success case build an explicit error if (!isHttpStatusSuccess(status)) { throw new ApiError( - `[Marketo Response Handler] - Request failed with status: ${status}`, + `[${destination} Response Handler] - Request failed with status: ${status}`, status, { scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.API.SCOPE, @@ -118,12 +122,12 @@ const marketoResponseHandler = ( }) ) { logger.info( - `[Marketo] Cache token evicting due to invalid/expired access_token for destinationId (${authKey})` + `${destination} Cache token evicting due to invalid/expired access_token for destinationId (${authKey})` ); authCache.del(authKey); } } - marketoApplicationErrorHandler(destResponse, sourceMessage, stage); + marketoApplicationErrorHandler(destResponse, sourceMessage, destination); } } // More readable error message diff --git a/v0/destinations/marketo_static_list/config.js b/v0/destinations/marketo_static_list/config.js new file mode 100644 index 00000000000..41bbab3548d --- /dev/null +++ b/v0/destinations/marketo_static_list/config.js @@ -0,0 +1,15 @@ +const MAX_LEAD_IDS_SIZE = 300; +const DESTINATION = "Marketo Static List"; + +const formatConfig = destination => { + return { + ID: destination.ID, + ...destination.Config + }; +}; + +module.exports = { + DESTINATION, + formatConfig, + MAX_LEAD_IDS_SIZE +}; diff --git a/v0/destinations/marketo_static_list/networkHandler.js b/v0/destinations/marketo_static_list/networkHandler.js new file mode 100644 index 00000000000..f9aeb080469 --- /dev/null +++ b/v0/destinations/marketo_static_list/networkHandler.js @@ -0,0 +1,46 @@ +const { TRANSFORMER_METRIC } = require("../../util/constant"); +const { marketoResponseHandler } = require("../marketo/util"); +const { + proxyRequest, + prepareProxyRequest +} = require("../../../adapters/network"); +const v0Utils = require("../../util"); +const { + processAxiosResponse +} = require("../../../adapters/utils/networkUtils"); +const { DESTINATION } = require("./config"); + +// eslint-disable-next-line no-unused-vars +const responseHandler = (destinationResponse, destType) => { + const message = `[Marketo Static List Response Handler] - Request Processed Successfully`; + const { status } = destinationResponse; + const authCache = v0Utils.getDestAuthCacheInstance(destType); + // check for marketo application level failures + marketoResponseHandler( + destinationResponse, + "during Marketo Static List Response Handling", + TRANSFORMER_METRIC.TRANSFORMER_STAGE.RESPONSE_TRANSFORM, + destinationResponse?.rudderJobMetadata, + authCache, + DESTINATION + ); + // else successfully return status, message and original destination response + return { + status, + message, + destinationResponse + }; +}; + +class networkHandler { + constructor() { + this.responseHandler = responseHandler; + this.proxy = proxyRequest; + this.prepareProxy = prepareProxyRequest; + this.processAxiosResponse = processAxiosResponse; + } +} + +module.exports = { + networkHandler +}; diff --git a/v0/destinations/marketo_static_list/transform.js b/v0/destinations/marketo_static_list/transform.js new file mode 100644 index 00000000000..fa07c220c81 --- /dev/null +++ b/v0/destinations/marketo_static_list/transform.js @@ -0,0 +1,225 @@ +const _ = require("lodash"); +const cloneDeep = require("lodash/cloneDeep"); +const { + defaultPostRequestConfig, + defaultDeleteRequestConfig +} = require("../../util"); +const { TRANSFORMER_METRIC, AUTH_CACHE_TTL } = require("../../util/constant"); +const { getIds, validateMessageType } = require("./util"); +const ErrorBuilder = require("../../util/error"); +const { + getDestinationExternalID, + defaultRequestConfig, + getErrorRespEvents, + generateErrorObject, + TransformationError, + simpleProcessRouterDest +} = require("../../util"); +const { DESTINATION, formatConfig, MAX_LEAD_IDS_SIZE } = require("./config"); +const Cache = require("../../util/cache"); +const logger = require("../../../logger"); +const { getAuthToken } = require("../marketo/transform"); + +const authCache = new Cache(AUTH_CACHE_TTL); // 1 hr + +const responseBuilder = (endPoint, leadIds, operation, token) => { + let updatedEndpoint = endPoint; + if (leadIds.length > 0) { + leadIds.forEach(id => { + updatedEndpoint = `${updatedEndpoint}id=${id}&`; + }); + } + updatedEndpoint = updatedEndpoint.slice(0, -1); + const response = defaultRequestConfig(); + response.endpoint = updatedEndpoint; + if (operation === "add") { + response.method = defaultPostRequestConfig.requestMethod; + } else { + response.method = defaultDeleteRequestConfig.requestMethod; + } + response.headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json" + }; + return response; +}; + +const batchResponseBuilder = (message, Config, token, leadIds, operation) => { + const { accountId } = Config; + const listId = + getDestinationExternalID(message, "marketoStaticListId") || + Config.staticListId; + const endpoint = `https://${accountId}.mktorest.com/rest/v1/lists/${listId}/leads.json?`; + if (!listId) { + throw new ErrorBuilder() + .setStatus(400) + .setMessage("No static listId is provided.") + .setStatTags({ + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.SCOPE, + meta: + TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.META + .INSTRUMENTATION + }) + .build(); + } + const response = []; + const leadIdsChunks = _.chunk(leadIds, MAX_LEAD_IDS_SIZE); + leadIdsChunks.forEach(ids => { + response.push(responseBuilder(endpoint, ids, operation, token)); + }); + return response; +}; + +const processEvent = input => { + const { token, message, destination } = input; + const { Config } = destination; + validateMessageType(message, ["audiencelist"]); + const response = []; + let toAdd; + let toRemove; + if (message.properties?.listData?.add) { + toAdd = getIds(message.properties.listData.add); + } + if (message.properties?.listData?.remove) { + toRemove = getIds(message.properties.listData.remove); + } + if ( + (Array.isArray(toAdd) && toAdd.length > 0) || + (Array.isArray(toRemove) && toRemove.length > 0) + ) { + if (Array.isArray(toAdd) && toAdd.length > 0) { + const payload = batchResponseBuilder( + message, + Config, + token, + toAdd, + "add" + ); + if (payload) { + response.push(...payload); + } + } + if (Array.isArray(toRemove) && toRemove.length > 0) { + const payload = batchResponseBuilder( + message, + Config, + token, + toRemove, + "remove" + ); + if (payload) { + response.push(...payload); + } + } + } else { + throw new ErrorBuilder() + .setMessage( + `Invalid leadIds format or no leadIds found neither to add nor to remove.` + ) + .setStatus(400) + .setStatTags({ + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.SCOPE, + meta: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.META.BAD_EVENT + }) + .build(); + } + return response; +}; +const process = async event => { + const token = await getAuthToken(formatConfig(event.destination)); + + if (!token) { + throw new TransformationError( + "Authorisation failed", + 400, + { + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.AUTHENTICATION.SCOPE + }, + DESTINATION + ); + } + const response = processEvent({ ...event, token }); + return response; +}; +const processRouterDest = async inputs => { + // Token needs to be generated for marketo which will be done on input level. + // If destination information is not present Error should be thrown + let token; + try { + token = await getAuthToken(formatConfig(inputs[0].destination)); + if (!token) { + const errResp = { + status: 400, + message: "Authorisation failed", + responseTransformFailure: true, + statTags: { + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.AUTHENTICATION.SCOPE + } + }; + const respEvents = getErrorRespEvents( + inputs.map(input => input.metadata), + errResp.status, + errResp.message, + errResp.statTags + ); + return [respEvents]; + } + } catch (error) { + // Not using handleRtTfSingleEventError here as this is for multiple events + logger.error("Router Transformation problem:"); + const errObj = generateErrorObject( + error, + DESTINATION, + TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM + ); + logger.error(errObj); + const respEvents = getErrorRespEvents( + inputs.map(input => input.metadata), + error.status || 500, // default to retryable + error.message || "Error occurred while processing payload.", + errObj.statTags + ); + return [respEvents]; + } + + // Checking previous status Code. Initially setting to false. + // If true then previous status is 500 and every subsequent event output should be + // sent with status code 500 to the router to be retried. + const tokenisedInputs = inputs.map(input => { + return { ...input, token }; + }); + const respList = await simpleProcessRouterDest( + tokenisedInputs, + DESTINATION, + processEvent + ); + return respList; +}; + +/** + * This function takes the transformed output containing metadata and returns the updated metadata + * @param {*} output + * @returns {*} metadata + */ +function processMetadataForRouter(output) { + const { metadata, destination } = output; + const clonedMetadata = cloneDeep(metadata); + clonedMetadata.forEach(metadataElement => { + // eslint-disable-next-line no-param-reassign + metadataElement.destInfo = { authKey: destination.ID }; + }); + return clonedMetadata; +} + +module.exports = { + process, + processRouterDest, + processMetadataForRouter, + authCache +}; diff --git a/v0/destinations/marketo_static_list/util.js b/v0/destinations/marketo_static_list/util.js new file mode 100644 index 00000000000..9cb4aa6e521 --- /dev/null +++ b/v0/destinations/marketo_static_list/util.js @@ -0,0 +1,60 @@ +const { DESTINATION } = require("./config"); +const { TRANSFORMER_METRIC } = require("../../util/constant"); +const ErrorBuilder = require("../../util/error"); + +/** + * Fetches the ids from the array of objects + * where each object has consist of Id + * @param {*} array + * @returns array of Ids + */ +const getIds = array => { + if (Array.isArray(array)) { + const leadIds = []; + if (array.length > 0) { + array.forEach(object => { + leadIds.push(object?.id); + }); + } + return leadIds; + } + return null; +}; + +/** + * Validates the message type and throws error if + * message type is not allowed or unavailable + * @param {*} message to get message type from + * @param {*} allowedTypes array of allowed message types + */ +const validateMessageType = (message, allowedTypes) => { + if (!message.type) { + throw new ErrorBuilder() + .setMessage("Message Type is not present. Aborting message.") + .setStatus(400) + .setStatTags({ + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.SCOPE, + meta: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.META.BAD_EVENT + }) + .build(); + } + if (!allowedTypes.includes(message.type.toLowerCase())) { + throw new ErrorBuilder() + .setMessage(`${message.type} call is not supported.`) + .setStatus(400) + .setStatTags({ + destType: DESTINATION, + stage: TRANSFORMER_METRIC.TRANSFORMER_STAGE.TRANSFORM, + scope: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.SCOPE, + meta: TRANSFORMER_METRIC.MEASUREMENT_TYPE.TRANSFORMATION.META.BAD_EVENT + }) + .build(); + } +}; + +module.exports = { + getIds, + validateMessageType +};