diff --git a/components/settings/Settings.tsx b/components/settings/Settings.tsx index 13d50ac..1be5a69 100644 --- a/components/settings/Settings.tsx +++ b/components/settings/Settings.tsx @@ -17,6 +17,7 @@ type SettingsError = { const defaultSettings = { scraper_type: 'none', + scrape_delay: 'none', notification_interval: 'daily', notification_email: '', smtp_server: '', @@ -105,6 +106,18 @@ const Settings = ({ closeSettings }:SettingsProps) => { { label: 'Monthly', value: 'monthly' }, { label: 'Never', value: 'never' }, ]; + const delayOptions: SelectionOption[] = [ + { label: 'No Delay', value: '0' }, + { label: '5 Seconds', value: '5000' }, + { label: '10 Seconds', value: '10000' }, + { label: '30 Seconds', value: '30000' }, + { label: '1 Minutes', value: '60000' }, + { label: '2 Minutes', value: '120000' }, + { label: '5 Minutes', value: '300000' }, + { label: '10 Minutes', value: '600000' }, + { label: '15 Minutes', value: '900000' }, + { label: '30 Minutes', value: '1800000' }, + ]; const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : []; const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers]; @@ -193,6 +206,20 @@ const Settings = ({ closeSettings }:SettingsProps) => { This option requires Server/Docker Instance Restart to take Effect. )} +
+ + updated[0] && updateSettings('scrape_delay', updated[0])} + rounded='rounded' + maxHeight={48} + minWidth={270} + /> + This option requires Server/Docker Instance Restart to take Effect. +
)} diff --git a/types.d.ts b/types.d.ts index 084b3ad..52ed0cd 100644 --- a/types.d.ts +++ b/types.d.ts @@ -79,7 +79,8 @@ type SettingsType = { smtp_password?: string, search_console_integrated?: boolean, available_scapers?: Array, - scrape_interval?: string + scrape_interval?: string, + scrape_delay?: string, version?: string } diff --git a/utils/refresh.ts b/utils/refresh.ts index 1471004..e4eef78 100644 --- a/utils/refresh.ts +++ b/utils/refresh.ts @@ -1,4 +1,5 @@ import { performance } from 'perf_hooks'; +import { setTimeout as sleep } from 'timers/promises'; import { RefreshResult, scrapeKeywordFromGoogle } from './scraper'; /** @@ -21,6 +22,9 @@ const refreshKeywords = async (keywords:KeywordType[], settings:SettingsType): P console.log('START SCRAPE: ', keyword.keyword); const refreshedkeywordData = await scrapeKeywordFromGoogle(keyword, settings); refreshedResults.push(refreshedkeywordData); + if (keywords.length > 0 && settings.scrape_delay && settings.scrape_delay !== '0') { + await sleep(parseInt(settings.scrape_delay, 10)); + } } }