From 925368a528ea434e68388cb0706523342e72ec74 Mon Sep 17 00:00:00 2001 From: hoseacodes Date: Thu, 16 May 2024 00:29:33 -0500 Subject: [PATCH] feat(data): add db backup to aws s3 --- .gitignore | 6 +- cron/backupDB.js | 147 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 111 ++++++++++++++++++++++++++++++++++ package.json | 1 + 4 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 cron/backupDB.js diff --git a/.gitignore b/.gitignore index ef649491..2e60dada 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,8 @@ src/.eslintrc.js test/coverage -animation \ No newline at end of file +animation + +dump + +output.json \ No newline at end of file diff --git a/cron/backupDB.js b/cron/backupDB.js new file mode 100644 index 00000000..89ac43ae --- /dev/null +++ b/cron/backupDB.js @@ -0,0 +1,147 @@ +import fs from "fs"; +import path from "path"; +import CronJob from "node-cron"; +import { exec } from "child_process"; +import AWS from "aws-sdk"; +import { stringToDate, empty, listFiles } from "../utils/helperFunctions.js"; + +// Auto backup function +// Only back up the database if there was a new deployment +export const dbAutoBackUp = async () => { + AWS.config.update({ + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }); + + const __dirname = path.resolve(path.dirname("")); + const backupDirPath = path.join(__dirname, "database-backup"); + const backupName = process.env.BACKUP_NAME; + const bucket = process.env.BACKUP_BUCKET_NAME; + const dumpPath = path.resolve(__dirname, backupName); + const s3 = new AWS.S3(); + + const dbOptions = { + user: process.env.DBUSER, + pass: process.env.DBPASSWORD, + host: process.env.DBHOST, + port: process.env.DBPORT, + database: process.env.DBNAME, + uri: process.env.MONGODB_URL || "mongodb://localhost:27017/portflio", + autoBackup: true, + removeOldBackup: false, + keepLastDaysBackup: 2, + autoBackupPath: backupDirPath, + }; + + // check for auto backup is enabled or disabled + if (dbOptions.autoBackup) { + let date = new Date(); + let beforeDate, oldBackupDir, oldBackupPath; + + // Current date + let currentDate = stringToDate(date); + let newBackupDir = + currentDate.getFullYear() + + "-" + + (currentDate.getMonth() + 1) + + "-" + + currentDate.getDate(); + + // New backup path for current backup process + let newBackupPath = dbOptions.autoBackupPath + "-mongodump-" + newBackupDir; + // check for remove old backup after keeping # of days given in configuration + if (dbOptions.removeOldBackup) { + beforeDate = _.clone(currentDate); + // Substract number of days to keep backup and remove old backup + beforeDate.setDate(beforeDate.getDate() - dbOptions.keepLastDaysBackup); + oldBackupDir = + beforeDate.getFullYear() + + "-" + + (beforeDate.getMonth() + 1) + + "-" + + beforeDate.getDate(); + // old backup(after keeping # of days) + oldBackupPath = dbOptions.autoBackupPath + "mongodump-" + oldBackupDir; + } + + let cmd = + "mongodump --uri=" + + dbOptions.uri + + ` --gzip --archive=${dumpPath} ` + + " --out " + + newBackupPath; + + try { + await exec(cmd, (error, stdout, stderr) => { + console.log({ error, stdout, stderr }); + if (empty(error)) { + // check for remove old backup after keeping # of days given in configuration. + if (dbOptions.removeOldBackup) { + if (fs.existsSync(oldBackupPath)) { + exec("rm -rf " + oldBackupPath, (err) => {}); + } + } + } + }); + + const backupFiles = [ + "articles.bson", + "articles.metadata.json", + "categories.bson", + "categories.metadata.json", + "comments.bson", + "comments.metadata.json", + "payments.bson", + "payments.metadata.json", + "products.bson", + "products.metadata.json", + "sessions.bson", + "sessions.metadata.json", + "users.bson", + "users.metadata.json", + ]; + // const readStream = fs.createReadStream(__dirname + "/dump/portflio/"); + // const params = { + // Bucket: bucket, + // Key: backupName, + // Body: readStream, + // }; + // await s3.putObject(params).promise(); + + backupFiles.forEach(async (file) => { + console.log(file); + const readStream = fs.createReadStream( + __dirname + "/dump/portflio/" + file + ); + const params = { + Bucket: bucket, + Key: `${backupName}-${file}`, + Body: readStream, + }; + await s3.putObject(params).promise(); + }); + + + console.log("Successful backup!"); + console.log("Backup path: ", __dirname + "/dump"); + } catch (error) { + console.log(`Backup failed: ${error}`); + } + } +}; + +// AutoBackUp every week (at 00:00 on Sunday) +export const initBackUpDBJob = () => { + const backUpJobFunction = CronJob.schedule( + "0 0 * * 0", + async () => { + dbAutoBackUp(); + }, + { + scheduled: true, + timezone: "America/Chicago", + } + ); + + backUpJobFunction.start(); +}; diff --git a/package-lock.json b/package-lock.json index 0402e937..24e644f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@use-effect/did-update": "^0.0.3", "@welldone-software/why-did-you-render": "^7.0.1", "aos": "^2.3.4", + "aws-sdk": "^2.1621.0", "axios": "^0.21.4", "bcrypt": "^5.0.1", "body-parser": "^1.19.0", @@ -9003,6 +9004,88 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-sdk": { + "version": "2.1621.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1621.0.tgz", + "integrity": "sha512-v6rxF1u0GpQG1Y9Wul9iaqulSV2uEnp0kHKd6/lZcvEgTYhtJ8N0hOLfqRSWHjH5PaIa46hR9zSAp51r8DJ/OA==", + "hasInstallScript": true, + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "util": "^0.12.4", + "uuid": "8.0.0", + "xml2js": "0.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-sdk/node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/aws-sdk/node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/aws-sdk/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + }, + "node_modules/aws-sdk/node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/aws-sdk/node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" + }, + "node_modules/aws-sdk/node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/aws-sdk/node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/axe-core": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz", @@ -20823,6 +20906,14 @@ "node": ">= 10.13.0" } }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/jpegtran-bin": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/jpegtran-bin/-/jpegtran-bin-5.0.2.tgz", @@ -34354,6 +34445,26 @@ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/package.json b/package.json index a74148d3..2d3e2bf8 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "@use-effect/did-update": "^0.0.3", "@welldone-software/why-did-you-render": "^7.0.1", "aos": "^2.3.4", + "aws-sdk": "^2.1621.0", "axios": "^0.21.4", "bcrypt": "^5.0.1", "body-parser": "^1.19.0",