-
Notifications
You must be signed in to change notification settings - Fork 409
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
AWS-SDK: Fixing broken exports #2410
Changes from 14 commits
dac4adf
98b8563
8338ac1
8cc52b8
43fd02d
5e31c04
3fcab79
af2bddd
6818d2a
0a52da7
78c2aaa
2c101dc
2e623b6
87538cf
fb0f56d
48f095e
243b610
052685b
7417b63
0d1055a
ee6a29c
be0bd30
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,7 +28,14 @@ import { rawIngestMethod } from "../extensions/contact-loaders"; | |
|
||
import { Lambda } from "@aws-sdk/client-lambda"; | ||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; | ||
import { GetObjectCommand, S3 } from "@aws-sdk/client-s3"; | ||
import { | ||
CreateBucketCommand, | ||
HeadBucketCommand, | ||
GetObjectCommand, | ||
waitUntilBucketExists, | ||
S3Client, | ||
PutObjectCommand | ||
} from "@aws-sdk/client-s3"; | ||
import { SQS } from "@aws-sdk/client-sqs"; | ||
import Papa from "papaparse"; | ||
import moment from "moment"; | ||
|
@@ -861,46 +868,106 @@ export async function exportCampaign(job) { | |
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) | ||
) { | ||
try { | ||
const s3bucket = new S3({ | ||
// The transformation for params is not implemented. | ||
// Refer to UPGRADING.md on aws-sdk-js-v3 for changes needed. | ||
// Please create/upvote feature request on aws-sdk-js-codemod for params. | ||
params: { Bucket: process.env.AWS_S3_BUCKET_NAME } | ||
const client = new S3Client({ | ||
region: process.env.AWS_REGION | ||
}); | ||
const bucketName = process.env.AWS_S3_BUCKET_NAME; | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we want to create a new bucket each time we export data, it's likely someone will already have bucket created for the data and want to upload directly to it we can add a check, if the bucket exists, save the exported data, if it doesn't, create a new one and save |
||
try { | ||
// Check if the S3 bucket already exists | ||
const verifyBucketCommand = new HeadBucketCommand({ | ||
Bucket: bucketName | ||
}); | ||
await client.send(verifyBucketCommand); | ||
|
||
console.log(`S3 bucket "${bucketName}" already exists.`); | ||
} catch (error) { | ||
if (error.name === "NotFound") { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When the bucket is not found, it doesn't look like we get this error code, only the error name |
||
console.log( | ||
`S3 bucket "${bucketName}" not found. Creating a new bucket.` | ||
); | ||
|
||
try { | ||
// Create the S3 bucket | ||
const createBucketCommand = new CreateBucketCommand({ | ||
Bucket: bucketName | ||
}); | ||
await client.send(createBucketCommand); | ||
|
||
console.log(`S3 bucket "${bucketName}" created successfully.`); | ||
} catch (createError) { | ||
console.error( | ||
`Error creating bucket "${bucketName}":`, | ||
createError | ||
); | ||
} | ||
} else { | ||
console.error("Error checking bucket existence:", error); | ||
} | ||
} | ||
|
||
// verifies that the bucket exists before moving forward | ||
// if for some reason this fails, Spoke defensively deletes the job | ||
await waitUntilBucketExists( | ||
{ client, maxWaitTime: 60 }, | ||
{ Bucket: bucketName } | ||
); | ||
|
||
const campaignTitle = campaign.title | ||
.replace(/ /g, "_") | ||
.replace(/\//g, "_"); | ||
const key = `${campaignTitle}-${moment().format( | ||
"YYYY-MM-DD-HH-mm-ss" | ||
)}.csv`; | ||
const messageKey = `${key}-messages.csv`; | ||
let params = { Key: key, Body: campaignCsv }; | ||
await s3bucket.putObject(params); | ||
params = { Key: key, Expires: 86400 }; | ||
const campaignExportUrl = await await getSignedUrl(s3bucket, new GetObjectCommand(params), { | ||
expiresIn: "/* add value from 'Expires' from v2 call if present, else remove */" | ||
}); | ||
params = { Key: messageKey, Body: messageCsv }; | ||
await s3bucket.putObject(params); | ||
params = { Key: messageKey, Expires: 86400 }; | ||
const campaignMessagesExportUrl = await await getSignedUrl(s3bucket, new GetObjectCommand(params), { | ||
expiresIn: "/* add value from 'Expires' from v2 call if present, else remove */" | ||
}); | ||
let params = { Key: key, | ||
Body: campaignCsv, | ||
Bucket: bucketName }; | ||
await client.send(new PutObjectCommand(params)); | ||
params = { Key: key, | ||
Expires: 86400, | ||
Bucket: bucketName }; | ||
const campaignExportUrl = await getSignedUrl(client, new GetObjectCommand(params)); | ||
params = { Key: messageKey, | ||
Body: messageCsv, | ||
Bucket: bucketName }; | ||
await client.send(new PutObjectCommand(params)); | ||
params = { Key: messageKey, | ||
Expires: 86400, | ||
Bucket: bucketName }; | ||
const campaignMessagesExportUrl = await getSignedUrl(client, new GetObjectCommand(params)); | ||
exportResults.campaignExportUrl = campaignExportUrl; | ||
exportResults.campaignMessagesExportUrl = campaignMessagesExportUrl; | ||
|
||
await sendEmail({ | ||
to: user.email, | ||
subject: `Export ready for ${campaign.title}`, | ||
text: `Your Spoke exports are ready! These URLs will be valid for 24 hours. | ||
Campaign export: ${campaignExportUrl} | ||
Message export: ${campaignMessagesExportUrl}` | ||
}).catch(err => { | ||
log.error(err); | ||
log.info(`Campaign Export URL - ${campaignExportUrl}`); | ||
log.info(`Campaign Messages Export URL - ${campaignMessagesExportUrl}`); | ||
}); | ||
log.info(`Successfully exported ${id}`); | ||
// extreme check on email set-up | ||
if (( | ||
process.env.EMAIL_FROM && | ||
process.env.EMAIL_HOST && | ||
process.env.EMAIL_HOST_PASSOWRD && | ||
process.env.EMAIL_HOST_PORT && | ||
process.env.EMAIL_HOST_USER) || | ||
( | ||
process.env.MAILGUN_DOMIAN && | ||
process.env.MAILGUN_SMTP_LOGN && | ||
process.env.MAILGUN_SMTP_PASSWORD && | ||
process.env.MAILGUN_SMTP_PORT && | ||
process.env.MAILGUN_SMTP_SERVER && | ||
process.env.MAILGUN_PUBLIC_KEY | ||
) | ||
) { | ||
await sendEmail({ | ||
to: user.email, | ||
subject: `Export ready for ${campaign.title}`, | ||
text: `Your Spoke exports are ready! These URLs will be valid for 24 hours. | ||
Campaign export: ${campaignExportUrl} | ||
Message export: ${campaignMessagesExportUrl}` | ||
}).catch(err => { | ||
log.error(err); | ||
log.info(`Campaign Export URL - ${campaignExportUrl}`); | ||
log.info(`Campaign Messages Export URL - ${campaignMessagesExportUrl}`); | ||
}); | ||
log.info(`Successfully exported ${id}`); | ||
} | ||
} catch (err) { | ||
log.error(err); | ||
exportResults.error = err.message; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I just tried this locally, if we have AWS S3 set up, even if email isn't enabled, the data will save to S3, not in the spoke directory
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
disregard my previous comment, the last change to this check fixed the issue I was seeing