diff --git a/database/migrations/1707068556345-add-new-keyword-fields.js b/database/migrations/1707068556345-add-new-keyword-fields.js index 5b9628b..f23edb0 100644 --- a/database/migrations/1707068556345-add-new-keyword-fields.js +++ b/database/migrations/1707068556345-add-new-keyword-fields.js @@ -2,18 +2,44 @@ // CLI Migration module.exports = { - up: (queryInterface, Sequelize) => { - return queryInterface.sequelize.transaction(async (t) => { - await queryInterface.addColumn('keyword', 'city', { type: Sequelize.DataTypes.STRING }, { transaction: t }); - await queryInterface.addColumn('keyword', 'latlong', { type: Sequelize.DataTypes.STRING }, { transaction: t }); - await queryInterface.addColumn('keyword', 'settings', { type: Sequelize.DataTypes.STRING }, { transaction: t }); - }); + up: async (queryInterface, Sequelize) => { + return queryInterface.sequelize.transaction(async (t) => { + try { + const keywordTableDefinition = await queryInterface.describeTable('keyword'); + if (keywordTableDefinition) { + if (!keywordTableDefinition.city) { + await queryInterface.addColumn('keyword', 'city', { type: Sequelize.DataTypes.STRING }, { transaction: t }); + } + if (!keywordTableDefinition.latlong) { + await queryInterface.addColumn('keyword', 'latlong', { type: Sequelize.DataTypes.STRING }, { transaction: t }); + } + if (!keywordTableDefinition.settings) { + await queryInterface.addColumn('keyword', 'settings', { type: Sequelize.DataTypes.STRING }, { transaction: t }); + } + } + } catch (error) { + console.log('error :', error); + } + }); }, down: (queryInterface) => { return queryInterface.sequelize.transaction(async (t) => { - await queryInterface.removeColumn('keyword', 'city', { transaction: t }); - await queryInterface.removeColumn('keyword', 'latlong', { transaction: t }); - await queryInterface.removeColumn('keyword', 'settings', { transaction: t }); + try { + const keywordTableDefinition = await queryInterface.describeTable('keyword'); + if (keywordTableDefinition) { + if (keywordTableDefinition.city) { + await queryInterface.removeColumn('keyword', 'city', { transaction: t }); + } + if (keywordTableDefinition.latlong) { + await queryInterface.removeColumn('keyword', 'latlong', { transaction: t }); + } + if (keywordTableDefinition.latlong) { + await queryInterface.removeColumn('keyword', 'settings', { transaction: t }); + } + } + } catch (error) { + console.log('error :', error); + } }); }, }; diff --git a/database/migrations/1707233039698-add-domain-searchconsole-field.js b/database/migrations/1707233039698-add-domain-searchconsole-field.js index 275907b..9213bbf 100644 --- a/database/migrations/1707233039698-add-domain-searchconsole-field.js +++ b/database/migrations/1707233039698-add-domain-searchconsole-field.js @@ -4,12 +4,26 @@ module.exports = { up: (queryInterface, Sequelize) => { return queryInterface.sequelize.transaction(async (t) => { - await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t }); + try { + const domainTableDefinition = await queryInterface.describeTable('domain'); + if (domainTableDefinition && !domainTableDefinition.search_console) { + await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t }); + } + } catch (error) { + console.log('error :', error); + } }); }, down: (queryInterface) => { return queryInterface.sequelize.transaction(async (t) => { - await queryInterface.removeColumn('domain', 'search_console', { transaction: t }); + try { + const domainTableDefinition = await queryInterface.describeTable('domain'); + if (domainTableDefinition && domainTableDefinition.search_console) { + await queryInterface.removeColumn('domain', 'search_console', { transaction: t }); + } + } catch (error) { + console.log('error :', error); + } }); }, }; diff --git a/package-lock.json b/package-lock.json index d92e47e..06e205f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,7 +37,8 @@ "reflect-metadata": "^0.1.13", "sequelize": "^6.34.0", "sequelize-typescript": "^2.1.6", - "sqlite3": "^5.1.6" + "sqlite3": "^5.1.6", + "umzug": "^3.6.1" }, "devDependencies": { "@testing-library/jest-dom": "^6.1.4", @@ -1931,6 +1932,25 @@ "integrity": "sha512-6i/8UoL0P5y4leBIGzvkZdS85RDMG9y1ihZzmTZQ5LdHUYmZ7pKFoj8X0236s3lusPs1Fa5HTQUpwI+UfTcmeA==", "dev": true }, + "node_modules/@rushstack/ts-command-line": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/@rushstack/ts-command-line/-/ts-command-line-4.17.1.tgz", + "integrity": "sha512-2jweO1O57BYP5qdBGl6apJLB+aRIn5ccIRTPDyULh0KMwVzFqWtw6IZWt1qtUoZD/pD2RNkIOosH6Cq45rIYeg==", + "dependencies": { + "@types/argparse": "1.0.38", + "argparse": "~1.0.9", + "colors": "~1.2.1", + "string-argv": "~0.3.1" + } + }, + "node_modules/@rushstack/ts-command-line/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -2075,6 +2095,11 @@ "node": ">= 10" } }, + "node_modules/@types/argparse": { + "version": "1.0.38", + "resolved": "https://registry.npmjs.org/@types/argparse/-/argparse-1.0.38.tgz", + "integrity": "sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==" + }, "node_modules/@types/aria-query": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.3.tgz", @@ -3819,6 +3844,14 @@ "dev": true, "peer": true }, + "node_modules/colors": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.2.5.tgz", + "integrity": "sha512-erNRLao/Y3Fv54qUa0LBB+//Uf3YwMUmdJinN20yMXm9zdKKqH9wt7R9IIVZ+K7ShzfpLV/Zg8+VyrBJYB4lpg==", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -5030,7 +5063,6 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, "engines": { "node": ">=12" }, @@ -11253,6 +11285,14 @@ "node": ">=8" } }, + "node_modules/pony-cause": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/pony-cause/-/pony-cause-2.1.10.tgz", + "integrity": "sha512-3IKLNXclQgkU++2fSi93sQ6BznFuxSLB11HdvZQ6JW/spahf/P1pAHBQEahr20rs0htZW0UDkM1HmA+nZkXKsw==", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/postcss": { "version": "8.4.31", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", @@ -12336,6 +12376,18 @@ "wrap-ansi": "^7.0.0" } }, + "node_modules/sequelize-cli/node_modules/umzug": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.3.0.tgz", + "integrity": "sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==", + "dev": true, + "dependencies": { + "bluebird": "^3.7.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/sequelize-cli/node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -12697,8 +12749,7 @@ "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" }, "node_modules/sqlite3": { "version": "5.1.6", @@ -12991,6 +13042,14 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/string-argv": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz", + "integrity": "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==", + "engines": { + "node": ">=0.6.19" + } + }, "node_modules/string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", @@ -14089,15 +14148,66 @@ } }, "node_modules/umzug": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.3.0.tgz", - "integrity": "sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==", - "dev": true, + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/umzug/-/umzug-3.6.1.tgz", + "integrity": "sha512-+ztJ2muIkP/Qw8w+4GfXEOrBZ/1kf9xlAyk9PSg2ZOpM5zX45vtQFDfuV18CdQLE1HWDL7EYT3Qcw0hFMWcp2Q==", "dependencies": { - "bluebird": "^3.7.2" + "@rushstack/ts-command-line": "^4.12.2", + "emittery": "^0.13.0", + "glob": "^8.0.3", + "pony-cause": "^2.1.4", + "type-fest": "^4.0.0" }, "engines": { - "node": ">=6.0.0" + "node": ">=12" + } + }, + "node_modules/umzug/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/umzug/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/umzug/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/umzug/node_modules/type-fest": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.10.2.tgz", + "integrity": "sha512-anpAG63wSpdEbLwOqH8L84urkL6PiVIov3EMmgIhhThevh9aiMQov+6Btx0wldNcvm4wV+e2/Rt1QdDwKHFbHw==", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/unbox-primitive": { diff --git a/package.json b/package.json index 53a0026..e9044b5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,6 @@ "build": "next build", "start": "next start", "cron": "node cron.js", - "prestart": "npm run db:migrate", "start:all": "concurrently npm:start npm:cron", "lint": "next lint", "lint:css": "stylelint styles/*.css", @@ -48,7 +47,8 @@ "reflect-metadata": "^0.1.13", "sequelize": "^6.34.0", "sequelize-typescript": "^2.1.6", - "sqlite3": "^5.1.6" + "sqlite3": "^5.1.6", + "umzug": "^3.6.1" }, "devDependencies": { "@testing-library/jest-dom": "^6.1.4", diff --git a/pages/api/dbmigrate.ts b/pages/api/dbmigrate.ts new file mode 100644 index 0000000..07b0848 --- /dev/null +++ b/pages/api/dbmigrate.ts @@ -0,0 +1,53 @@ +import { Sequelize } from 'sequelize'; +import { Umzug, SequelizeStorage } from 'umzug'; +import type { NextApiRequest, NextApiResponse } from 'next'; +import db from '../../database/database'; +import verifyUser from '../../utils/verifyUser'; + +type MigrationGetResponse = { + hasMigrations: boolean, +} + +type MigrationPostResponse = { + migrated: boolean, + erroor?: string +} + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + const authorized = verifyUser(req, res); + if (authorized === 'authorized' && req.method === 'GET') { + await db.sync(); + return getMigrationStatus(req, res); + } + if (authorized === 'authorized' && req.method === 'POST') { + return migrateDatabase(req, res); + } + return res.status(401).json({ error: authorized }); +} + +const getMigrationStatus = async (req: NextApiRequest, res: NextApiResponse) => { + const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false }); + const umzug = new Umzug({ + migrations: { glob: 'database/migrations/*.js' }, + context: sequelize.getQueryInterface(), + storage: new SequelizeStorage({ sequelize }), + logger: undefined, + }); + const migrations = await umzug.pending(); + // console.log('migrations :', migrations); + // const migrationsExceuted = await umzug.executed(); + return res.status(200).json({ hasMigrations: migrations.length > 0 }); +}; + +const migrateDatabase = async (req: NextApiRequest, res: NextApiResponse) => { + const sequelize = new Sequelize({ dialect: 'sqlite', storage: './data/database.sqlite', logging: false }); + const umzug = new Umzug({ + migrations: { glob: 'database/migrations/*.js' }, + context: sequelize.getQueryInterface(), + storage: new SequelizeStorage({ sequelize }), + logger: undefined, + }); + const migrations = await umzug.up(); + console.log('[Updated] migrations :', migrations); + return res.status(200).json({ migrated: true }); +}; diff --git a/pages/domains/index.tsx b/pages/domains/index.tsx index 2bccabc..7bff710 100644 --- a/pages/domains/index.tsx +++ b/pages/domains/index.tsx @@ -7,7 +7,7 @@ import toast, { Toaster } from 'react-hot-toast'; import TopBar from '../../components/common/TopBar'; import AddDomain from '../../components/domains/AddDomain'; import Settings from '../../components/settings/Settings'; -import { useFetchSettings } from '../../services/settings'; +import { useCheckMigrationStatus, useFetchSettings } from '../../services/settings'; import { fetchDomainScreenshot, useFetchDomains } from '../../services/domains'; import DomainItem from '../../components/domains/DomainItem'; import Icon from '../../components/common/Icon'; @@ -22,6 +22,9 @@ const Domains: NextPage = () => { const [domainThumbs, setDomainThumbs] = useState({}); const { data: appSettingsData, isLoading: isAppSettingsLoading } = useFetchSettings(); const { data: domainsData, isLoading } = useFetchDomains(router, true); + const { data: migrationStatus } = useCheckMigrationStatus(); + // const { mutate: updateDatabaseMutate, isLoading: isUpdatingDB } = useMigrateDatabase((res:Object) => { window.location.reload(); }); + const appSettings:SettingsType = appSettingsData?.settings || {}; const { scraper_type = '' } = appSettings; @@ -78,6 +81,12 @@ const Domains: NextPage = () => { A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app. )} + {migrationStatus?.hasMigrations && ( +
+ You need to Update your database. Stop Serpbear and run this command to update your database: + npm run db:migrate +
+ )} Domains - SerpBear diff --git a/services/settings.ts b/services/settings.ts index fc21a77..c858cef 100644 --- a/services/settings.ts +++ b/services/settings.ts @@ -60,3 +60,37 @@ export function useClearFailedQueue(onSuccess:Function) { }, }); } + +export async function fetchMigrationStatus() { + const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'GET' }); + return res.json(); +} + +export function useCheckMigrationStatus() { + return useQuery('dbmigrate', () => fetchMigrationStatus()); +} + +export const useMigrateDatabase = (onSuccess:Function|undefined) => { + const queryClient = useQueryClient(); + + return useMutation(async () => { + // console.log('settings: ', JSON.stringify(settings)); + const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'POST' }); + if (res.status >= 400 && res.status < 600) { + throw new Error('Bad response from server'); + } + return res.json(); + }, { + onSuccess: async (res) => { + if (onSuccess) { + onSuccess(res); + } + toast('Database Updated!', { icon: '✔️' }); + queryClient.invalidateQueries(['settings']); + }, + onError: () => { + console.log('Error Updating Database!!!'); + toast('Error Updating Database.', { icon: '⚠️' }); + }, + }); +};