Skip to content

Commit

Permalink
feat(platform): add support for job retries (only constant strategy)
Browse files Browse the repository at this point in the history
  • Loading branch information
@azasypkin
azasypkin committed Dec 5, 2023
1 parent 4524079 commit b44fd2d
Show file tree
Hide file tree
Showing 5 changed files with 292 additions and 60 deletions.
56 changes: 56 additions & 0 deletions src/pages/workspace/utils/web_scraping/consts.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,63 @@
import type { SchedulerJobRetryStrategy } from './web_page_tracker';

export const WEB_PAGE_TRACKER_SCHEDULES = [
{ value: '@', text: 'Manually' },
{ value: '@hourly', text: 'Hourly' },
{ value: '@daily', text: 'Daily' },
{ value: '@weekly', text: 'Weekly' },
{ value: '@monthly', text: 'Monthly' },
];

export const WEB_PAGE_TRACKER_RETRY_STRATEGIES = [
{ value: 'none', text: 'None' },
{ value: 'constant', text: 'Constant backoff' },
];

export const WEB_PAGE_TRACKER_RETRY_INTERVALS = new Map([
[
'@hourly',
[
{ label: '1m', value: 60000 },
{ label: '3m', value: 180000 },
{ label: '5m', value: 300000 },
{ label: '10m', value: 600000 },
],
],
[
'@daily',
[
{ label: '30m', value: 1800000 },
{ label: '1h', value: 3600000 },
{ label: '2h', value: 7200000 },
{ label: '3h', value: 10800000 },
],
],
[
'@weekly',
[
{ label: '1h', value: 3600000 },
{ label: '3h', value: 10800000 },
{ label: '6h', value: 21600000 },
{ label: '12h', value: 43200000 },
],
],
[
'@monthly',
[
{ label: '3h', value: 10800000 },
{ label: '12h', value: 43200000 },
{ label: '1d', value: 86400000 },
{ label: '3d', value: 259200000 },
],
],
]);

export function getDefaultRetryStrategy(schedule: string): SchedulerJobRetryStrategy {
return { type: 'constant', maxAttempts: 3, interval: getDefaultRetryInterval(schedule) };
}

// By default, use the middle interval, e.g. 5 minutes for hourly schedule.
export function getDefaultRetryInterval(schedule: string) {
const intervals = WEB_PAGE_TRACKER_RETRY_INTERVALS.get(schedule)!;
return intervals[Math.floor(intervals.length / 2)].value;
}
92 changes: 62 additions & 30 deletions src/pages/workspace/utils/web_scraping/web_page_content_tracker_edit_flyout.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ import {
EuiForm,
EuiFormRow,
EuiLink,
EuiRange,
EuiSelect,
EuiSwitch,
} from '@elastic/eui';
import type { EuiSwitchEvent } from '@elastic/eui';
import axios from 'axios';

import { WEB_PAGE_TRACKER_SCHEDULES } from './consts';
import type { WebPageContentTracker } from './web_page_tracker';
import { getDefaultRetryStrategy, WEB_PAGE_TRACKER_SCHEDULES } from './consts';
import type { SchedulerJobConfig, WebPageContentTracker } from './web_page_tracker';
import { WebPageTrackerRetryStrategy } from './web_page_tracker_retry_strategy';
import WebPageTrackerScriptEditor from './web_page_tracker_script_editor';
import { type AsyncData, getApiRequestConfig, getApiUrl, getErrorMessage, isClientError } from '../../../../model';
import { isValidURL } from '../../../../tools/url';
Expand Down Expand Up @@ -45,10 +46,7 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
setUrl(e.target.value);
}, []);

const [sendNotification, setSendNotification] = useState<boolean>(true);
const onSendNotificationChange = useCallback((e: EuiSwitchEvent) => {
setSendNotification(e.target.checked);
}, []);
const [jobConfig, setJobConfig] = useState<SchedulerJobConfig | null>(tracker?.jobConfig ?? null);

const [delay, setDelay] = useState<number>(tracker?.settings.delay ?? 5000);
const onDelayChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
Expand All @@ -62,11 +60,6 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
invalid: false,
});

const [schedule, setSchedule] = useState<string>(tracker?.settings.schedule ?? '@');
const onScheduleChange = useCallback((e: ChangeEvent<HTMLSelectElement>) => {
setSchedule(e.target.value);
}, []);

const [extractContentScript, setExtractContentScript] = useState<string | undefined>(
tracker?.settings.scripts?.extractContent,
);
Expand All @@ -75,9 +68,6 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
}, []);

const [revisions, setRevisions] = useState<number>(tracker?.settings.revisions ?? 3);
const onRevisionsChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
setRevisions(+e.target.value);
}, []);

const [updatingStatus, setUpdatingStatus] = useState<AsyncData<void>>();
const onSave = useCallback(() => {
Expand All @@ -93,7 +83,6 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
settings: {
revisions,
delay,
schedule: schedule === '@' ? undefined : schedule,
scripts: extractContentScript ? { extractContent: extractContentScript } : undefined,
headers:
headers.values.length > 0
Expand All @@ -107,8 +96,8 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
}),
)
: undefined,
enableNotifications: sendNotification,
},
jobConfig: jobConfig ? jobConfig : tracker?.jobConfig ? null : undefined,
};

const [requestPromise, successMessage, errorMessage] = tracker
Expand Down Expand Up @@ -147,7 +136,21 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
});
},
);
}, [name, url, delay, revisions, schedule, headers, extractContentScript, sendNotification, updatingStatus]);
}, [name, url, delay, revisions, headers, extractContentScript, jobConfig, tracker, updatingStatus]);

const notifications = jobConfig ? (
<EuiFormRow
label={'Notifications'}
helpText={'Send an email notification when a change is detected or a check fails.'}
>
<EuiSwitch
showLabel={false}
label="Notification on change"
checked={jobConfig.notifications}
onChange={(e) => setJobConfig({ ...jobConfig, notifications: e.target.checked })}
/>
</EuiFormRow>
) : null;

return (
<EditorFlyout
Expand All @@ -166,7 +169,15 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
<EuiFieldText value={url} required type={'url'} onChange={onUrlChange} />
</EuiFormRow>
<EuiFormRow label="Revisions" helpText="Tracker will persist only specified number of revisions">
<EuiFieldNumber fullWidth min={0} max={10} step={1} value={revisions} onChange={onRevisionsChange} />
<EuiRange
min={0}
max={10}
step={1}
value={revisions}
fullWidth
onChange={(e) => setRevisions(+e.currentTarget.value)}
showTicks
/>
</EuiFormRow>
<EuiFormRow
label="Delay"
Expand Down Expand Up @@ -212,20 +223,41 @@ export function WebPageContentTrackerEditFlyout({ onClose, tracker }: Props) {
label="Frequency"
helpText="How often web page should be checked for changes. By default, automatic checks are disabled and can be initiated manually"
>
<EuiSelect options={WEB_PAGE_TRACKER_SCHEDULES} value={schedule} onChange={onScheduleChange} />
</EuiFormRow>
<EuiFormRow
label={'Notifications'}
helpText={"Send notification to user's primary email when a change is detected"}
>
<EuiSwitch
showLabel={false}
label="Notification on change"
checked={sendNotification}
onChange={onSendNotificationChange}
<EuiSelect
options={WEB_PAGE_TRACKER_SCHEDULES}
value={jobConfig?.schedule ?? '@'}
onChange={(e) =>
setJobConfig(
e.target.value === '@'
? null
: {
...(jobConfig ?? {
retryStrategy: getDefaultRetryStrategy(e.target.value),
notifications: true,
}),
schedule: e.target.value,
},
)
}
/>
</EuiFormRow>
{notifications}
</EuiDescribedFormGroup>
{jobConfig ? (
<EuiDescribedFormGroup
title={<h3>Retries</h3>}
description={'Properties defining how failed automatic checks should be retried'}
>
<WebPageTrackerRetryStrategy
jobConfig={jobConfig}
onChange={(newStrategy) => {
if (jobConfig) {
setJobConfig({ ...jobConfig, retryStrategy: newStrategy ?? undefined });
}
}}
/>
</EuiDescribedFormGroup>
) : null}
<EuiDescribedFormGroup
title={<h3>Scripts</h3>}
description={'Custom JavaScript scripts that will be injected into the web page before content is extracted'}
Expand Down
92 changes: 62 additions & 30 deletions src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ import {
EuiForm,
EuiFormRow,
EuiLink,
EuiRange,
EuiSelect,
EuiSwitch,
} from '@elastic/eui';
import type { EuiSwitchEvent } from '@elastic/eui';
import axios from 'axios';

import { WEB_PAGE_TRACKER_SCHEDULES } from './consts';
import type { WebPageResourcesTracker } from './web_page_tracker';
import { getDefaultRetryStrategy, WEB_PAGE_TRACKER_SCHEDULES } from './consts';
import type { SchedulerJobConfig, WebPageResourcesTracker } from './web_page_tracker';
import { WebPageTrackerRetryStrategy } from './web_page_tracker_retry_strategy';
import WebPageTrackerScriptEditor from './web_page_tracker_script_editor';
import { type AsyncData, getApiRequestConfig, getApiUrl, getErrorMessage, isClientError } from '../../../../model';
import { isValidURL } from '../../../../tools/url';
Expand Down Expand Up @@ -45,10 +46,7 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
setUrl(e.target.value);
}, []);

const [sendNotification, setSendNotification] = useState<boolean>(true);
const onSendNotificationChange = useCallback((e: EuiSwitchEvent) => {
setSendNotification(e.target.checked);
}, []);
const [jobConfig, setJobConfig] = useState<SchedulerJobConfig | null>(tracker?.jobConfig ?? null);

const [delay, setDelay] = useState<number>(tracker?.settings.delay ?? 5000);
const onDelayChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
Expand All @@ -62,11 +60,6 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
invalid: false,
});

const [schedule, setSchedule] = useState<string>(tracker?.settings.schedule ?? '@');
const onScheduleChange = useCallback((e: ChangeEvent<HTMLSelectElement>) => {
setSchedule(e.target.value);
}, []);

const [resourceFilterMapScript, setResourceFilterMapScript] = useState<string | undefined>(
tracker?.settings.scripts?.resourceFilterMap,
);
Expand All @@ -75,9 +68,6 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
}, []);

const [revisions, setRevisions] = useState<number>(tracker?.settings.revisions ?? 3);
const onRevisionsChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
setRevisions(+e.target.value);
}, []);

const [updatingStatus, setUpdatingStatus] = useState<AsyncData<void>>();
const onSave = useCallback(() => {
Expand All @@ -93,7 +83,6 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
settings: {
revisions,
delay,
schedule: schedule === '@' ? undefined : schedule,
scripts: resourceFilterMapScript ? { resourceFilterMap: resourceFilterMapScript } : undefined,
headers:
headers.values.length > 0
Expand All @@ -107,8 +96,8 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
}),
)
: undefined,
enableNotifications: sendNotification,
},
jobConfig: jobConfig ? jobConfig : tracker?.jobConfig ? null : undefined,
};

const [requestPromise, successMessage, errorMessage] = tracker
Expand Down Expand Up @@ -151,7 +140,21 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
});
},
);
}, [name, url, delay, revisions, schedule, resourceFilterMapScript, headers, sendNotification, updatingStatus]);
}, [name, url, delay, revisions, resourceFilterMapScript, headers, jobConfig, tracker, updatingStatus]);

const notifications = jobConfig ? (
<EuiFormRow
label={'Notifications'}
helpText={'Send an email notification when a change is detected or a check fails.'}
>
<EuiSwitch
showLabel={false}
label="Notification on change"
checked={jobConfig.notifications}
onChange={(e) => setJobConfig({ ...jobConfig, notifications: e.target.checked })}
/>
</EuiFormRow>
) : null;

return (
<EditorFlyout
Expand All @@ -170,7 +173,15 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
<EuiFieldText value={url} required type={'url'} onChange={onUrlChange} />
</EuiFormRow>
<EuiFormRow label="Revisions" helpText="Tracker will persist only specified number of revisions">
<EuiFieldNumber fullWidth min={0} max={10} step={1} value={revisions} onChange={onRevisionsChange} />
<EuiRange
min={0}
max={10}
step={1}
value={revisions}
fullWidth
onChange={(e) => setRevisions(+e.currentTarget.value)}
showTicks
/>
</EuiFormRow>
<EuiFormRow
label="Delay"
Expand Down Expand Up @@ -216,20 +227,41 @@ export function WebPageResourcesTrackerEditFlyout({ onClose, tracker }: Props) {
label="Frequency"
helpText="How often web page should be checked for changes. By default, automatic checks are disabled and can be initiated manually"
>
<EuiSelect options={WEB_PAGE_TRACKER_SCHEDULES} value={schedule} onChange={onScheduleChange} />
</EuiFormRow>
<EuiFormRow
label={'Notifications'}
helpText={"Send notification to user's primary email when a change is detected"}
>
<EuiSwitch
showLabel={false}
label="Notification on change"
checked={sendNotification}
onChange={onSendNotificationChange}
<EuiSelect
options={WEB_PAGE_TRACKER_SCHEDULES}
value={jobConfig?.schedule ?? '@'}
onChange={(e) =>
setJobConfig(
e.target.value === '@'
? null
: {
...(jobConfig ?? {
retryStrategy: getDefaultRetryStrategy(e.target.value),
notifications: true,
}),
schedule: e.target.value,
},
)
}
/>
</EuiFormRow>
{notifications}
</EuiDescribedFormGroup>
{jobConfig ? (
<EuiDescribedFormGroup
title={<h3>Retries</h3>}
description={'Properties defining how failed automatic checks should be retried'}
>
<WebPageTrackerRetryStrategy
jobConfig={jobConfig}
onChange={(newStrategy) => {
if (jobConfig) {
setJobConfig({ ...jobConfig, retryStrategy: newStrategy ?? undefined });
}
}}
/>
</EuiDescribedFormGroup>
) : null}
<EuiDescribedFormGroup
title={<h3>Scripts</h3>}
description={
Expand Down
Loading

0 comments on commit b44fd2d

Please sign in to comment.