Skip to content

Commit

Permalink
feat: Add option to Delay Between scrapes.
Browse files Browse the repository at this point in the history
fixes #87
  • Loading branch information
towfiqi committed Mar 29, 2023
1 parent d950515 commit 0a83924
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 1 deletion.
27 changes: 27 additions & 0 deletions components/settings/Settings.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type SettingsError = {

const defaultSettings = {
scraper_type: 'none',
scrape_delay: 'none',
notification_interval: 'daily',
notification_email: '',
smtp_server: '',
Expand Down Expand Up @@ -105,6 +106,18 @@ const Settings = ({ closeSettings }:SettingsProps) => {
{ label: 'Monthly', value: 'monthly' },
{ label: 'Never', value: 'never' },
];
const delayOptions: SelectionOption[] = [
{ label: 'No Delay', value: '0' },
{ label: '5 Seconds', value: '5000' },
{ label: '10 Seconds', value: '10000' },
{ label: '30 Seconds', value: '30000' },
{ label: '1 Minutes', value: '60000' },
{ label: '2 Minutes', value: '120000' },
{ label: '5 Minutes', value: '300000' },
{ label: '10 Minutes', value: '600000' },
{ label: '15 Minutes', value: '900000' },
{ label: '30 Minutes', value: '1800000' },
];
const allScrapers: SelectionOption[] = settings.available_scapers ? settings.available_scapers : [];
const scraperOptions: SelectionOption[] = [{ label: 'None', value: 'none' }, ...allScrapers];

Expand Down Expand Up @@ -193,6 +206,20 @@ const Settings = ({ closeSettings }:SettingsProps) => {
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
</div>
)}
<div className="settings__section__input mb-5">
<label className={labelStyle}>Delay Between Each keyword Scrape</label>
<SelectField
multiple={false}
selected={[settings?.scrape_delay || '0']}
options={delayOptions}
defaultLabel={'Delay Settings'}
updateField={(updated:string[]) => updated[0] && updateSettings('scrape_delay', updated[0])}
rounded='rounded'
maxHeight={48}
minWidth={270}
/>
<small className=' text-gray-500 pt-2 block'>This option requires Server/Docker Instance Restart to take Effect.</small>
</div>
</div>
</div>
)}
Expand Down
3 changes: 2 additions & 1 deletion types.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ type SettingsType = {
smtp_password?: string,
search_console_integrated?: boolean,
available_scapers?: Array,
scrape_interval?: string
scrape_interval?: string,
scrape_delay?: string,
version?: string
}

Expand Down
4 changes: 4 additions & 0 deletions utils/refresh.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { performance } from 'perf_hooks';
import { setTimeout as sleep } from 'timers/promises';
import { RefreshResult, scrapeKeywordFromGoogle } from './scraper';

/**
Expand All @@ -21,6 +22,9 @@ const refreshKeywords = async (keywords:KeywordType[], settings:SettingsType): P
console.log('START SCRAPE: ', keyword.keyword);
const refreshedkeywordData = await scrapeKeywordFromGoogle(keyword, settings);
refreshedResults.push(refreshedkeywordData);
if (keywords.length > 0 && settings.scrape_delay && settings.scrape_delay !== '0') {
await sleep(parseInt(settings.scrape_delay, 10));
}
}
}

Expand Down

0 comments on commit 0a83924

Please sign in to comment.