[Feature] Add support for custom resource tracker filter/map scripts.

secutils-dev · Sep 7, 2023 · f485ff1 · f485ff1
1 parent c6894f8
commit f485ff1
Show file tree

Hide file tree

Showing 5 changed files with 184 additions and 3 deletions.
diff --git a/package.json b/package.json
@@ -27,6 +27,7 @@
     "@elastic/eui": "^88.2.0",
     "@emotion/css": "^11.11.2",
     "@emotion/react": "^11.11.1",
+    "@monaco-editor/react": "^4.5.2",
     "axios": "^1.5.0",
     "moment": "^2.29.4",
     "react": "^18.2.0",
@@ -59,6 +60,7 @@
     "eslint-plugin-import": "^2.28.1",
     "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-react": "^7.33.2",
+    "monaco-editor": "^0.41.0",
     "parcel": "^2.9.3",
     "prettier": "^3.0.3",
     "process": "^0.11.10",

diff --git a/src/pages/workspace/utils/web_scraping/web_page_resources_tracker.ts b/src/pages/workspace/utils/web_scraping/web_page_resources_tracker.ts
@@ -8,4 +8,7 @@ export interface WebPageResourcesTracker {
   revisions: number;
   delay: number;
   schedule?: string;
+  scripts?: {
+    resourceFilterMap?: string;
+  };
 }
diff --git a/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx b/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_edit_flyout.tsx
@@ -1,11 +1,20 @@
 import type { ChangeEvent } from 'react';
 import { useCallback, useState } from 'react';
 
-import { EuiDescribedFormGroup, EuiFieldNumber, EuiFieldText, EuiForm, EuiFormRow, EuiSelect } from '@elastic/eui';
+import {
+  EuiDescribedFormGroup,
+  EuiFieldNumber,
+  EuiFieldText,
+  EuiForm,
+  EuiFormRow,
+  EuiLink,
+  EuiSelect,
+} from '@elastic/eui';
 import axios from 'axios';
 
 import type { WebPageResourcesTracker, WebPageResourcesTrackers } from './web_page_resources_tracker';
 import { WEB_PAGE_RESOURCES_TRACKERS_USER_DATA_NAMESPACE } from './web_page_resources_tracker';
+import WebScrapingResourcesTrackerScriptEditor from './web_page_resources_tracker_script_editor';
 import { type AsyncData, getApiUrl, getErrorMessage, getUserData } from '../../../../model';
 import { EditorFlyout } from '../../components/editor_flyout';
 import { useWorkspaceContext } from '../../hooks';
@@ -55,6 +64,13 @@ export function WebScrapingResourcesTrackerEditFlyout({ onClose, item }: Props)
     setSchedule(e.target.value);
   }, []);
 
+  const [resourceFilterMapScript, setResourceFilterMapScript] = useState<string | undefined>(
+    item?.scripts?.resourceFilterMap,
+  );
+  const onResourceFilterMapScriptChange = useCallback((value?: string) => {
+    setResourceFilterMapScript(value);
+  }, []);
+
   const [revisions, setRevisions] = useState<number>(item?.revisions ?? 3);
   const onRevisionsChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
     setRevisions(+e.target.value);
@@ -74,7 +90,16 @@ export function WebScrapingResourcesTrackerEditFlyout({ onClose, item }: Props)
           type: 'webScraping',
           value: {
             type: 'saveWebPageResourcesTracker',
-            value: { tracker: { name, url, revisions, delay, schedule: schedule === '@' ? undefined : schedule } },
+            value: {
+              tracker: {
+                name,
+                url,
+                revisions,
+                delay,
+                schedule: schedule === '@' ? undefined : schedule,
+                scripts: resourceFilterMapScript ? { resourceFilterMap: resourceFilterMapScript } : undefined,
+              },
+            },
           },
         },
       })
@@ -103,7 +128,7 @@ export function WebScrapingResourcesTrackerEditFlyout({ onClose, item }: Props)
           });
         },
       );
-  }, [name, url, delay, revisions, schedule, updatingStatus]);
+  }, [name, url, delay, revisions, schedule, resourceFilterMapScript, updatingStatus]);
 
   return (
     <EditorFlyout
@@ -139,6 +164,31 @@ export function WebScrapingResourcesTrackerEditFlyout({ onClose, item }: Props)
             <EuiFieldNumber fullWidth min={0} max={60000} step={1000} value={delay} onChange={onDelayChange} />
           </EuiFormRow>
         </EuiDescribedFormGroup>
+        <EuiDescribedFormGroup
+          title={<h3>Scripts</h3>}
+          description={
+            'Custom JavaScript scripts that will be injected into the web page before resources are extracted'
+          }
+        >
+          <EuiFormRow
+            label="Resource filter/mapper"
+            helpText={
+              <span>
+                The script accepts "resource" as an argument and returns it, either with or without modifications, if
+                the resource should be tracked, or "null" if it should not. Refer to the{' '}
+                <EuiLink target="_blank" href="/docs/guides/web_scraping/resources">
+                  <b>documentation</b>
+                </EuiLink>{' '}
+                for a list of available "resource" properties and script examples.
+              </span>
+            }
+          >
+            <WebScrapingResourcesTrackerScriptEditor
+              onChange={onResourceFilterMapScriptChange}
+              defaultValue={resourceFilterMapScript}
+            />
+          </EuiFormRow>
+        </EuiDescribedFormGroup>
       </EuiForm>
     </EditorFlyout>
   );

diff --git a/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_script_editor.tsx b/src/pages/workspace/utils/web_scraping/web_page_resources_tracker_script_editor.tsx
@@ -0,0 +1,102 @@
+import { useEuiTheme } from '@elastic/eui';
+import { default as v8Dark } from '@elastic/eui/dist/eui_theme_dark.json';
+import { default as v8Light } from '@elastic/eui/dist/eui_theme_light.json';
+import { Editor, loader } from '@monaco-editor/react';
+import type { editor } from 'monaco-editor';
+import * as monaco from 'monaco-editor';
+
+loader.config({ monaco });
+
+export function createTheme(
+  euiTheme: typeof v8Dark | typeof v8Light,
+  selectionBackgroundColor: string,
+  backgroundColor?: string,
+): editor.IStandaloneThemeData {
+  return {
+    base: 'vs',
+    inherit: true,
+    rules: [
+      {
+        token: '',
+        foreground: euiTheme.euiColorDarkestShade,
+        background: euiTheme.euiFormBackgroundColor,
+      },
+      { token: 'invalid', foreground: euiTheme.euiColorAccent },
+      { token: 'emphasis', fontStyle: 'italic' },
+      { token: 'strong', fontStyle: 'bold' },
+
+      { token: 'variable', foreground: euiTheme.euiColorPrimary },
+      { token: 'variable.predefined', foreground: euiTheme.euiColorSuccess },
+      { token: 'constant', foreground: euiTheme.euiColorAccent },
+      { token: 'comment', foreground: euiTheme.euiColorMediumShade },
+      { token: 'number', foreground: euiTheme.euiColorAccent },
+      { token: 'number.hex', foreground: euiTheme.euiColorAccent },
+      { token: 'regexp', foreground: euiTheme.euiColorDanger },
+      { token: 'annotation', foreground: euiTheme.euiColorMediumShade },
+      { token: 'type', foreground: euiTheme.euiColorVis0 },
+
+      { token: 'delimiter', foreground: euiTheme.euiTextSubduedColor },
+      { token: 'delimiter.parenthesis', foreground: euiTheme.euiTextSubduedColor },
+
+      { token: 'tag', foreground: euiTheme.euiColorDanger },
+      { token: 'metatag', foreground: euiTheme.euiColorSuccess },
+
+      { token: 'key', foreground: euiTheme.euiColorWarning },
+
+      { token: 'attribute.name', foreground: euiTheme.euiColorDanger },
+      { token: 'attribute.value', foreground: euiTheme.euiColorPrimary },
+      { token: 'attribute.value.number', foreground: euiTheme.euiColorWarning },
+      { token: 'attribute.value.unit', foreground: euiTheme.euiColorWarning },
+
+      { token: 'string', foreground: euiTheme.euiColorDanger },
+
+      { token: 'keyword', foreground: euiTheme.euiColorPrimary },
+      { token: 'keyword.deprecated', foreground: euiTheme.euiColorAccent },
+
+      { token: 'text', foreground: euiTheme.euiTitleColor },
+      { token: 'label', foreground: euiTheme.euiColorVis9 },
+    ],
+    colors: {
+      'editor.foreground': euiTheme.euiColorDarkestShade,
+      'editor.background': backgroundColor ?? euiTheme.euiFormBackgroundColor,
+      'editorLineNumber.foreground': euiTheme.euiColorDarkShade,
+      'editorLineNumber.activeForeground': euiTheme.euiColorDarkShade,
+      'editorIndentGuide.background': euiTheme.euiColorLightShade,
+      'editor.selectionBackground': selectionBackgroundColor,
+      'editorWidget.border': euiTheme.euiColorLightShade,
+      'editorWidget.background': euiTheme.euiColorLightestShade,
+      'editorCursor.foreground': euiTheme.euiColorDarkestShade,
+      'editorSuggestWidget.selectedBackground': euiTheme.euiColorLightShade,
+      'list.hoverBackground': euiTheme.euiColorLightShade,
+      'list.highlightForeground': euiTheme.euiColorPrimary,
+      'editor.lineHighlightBorder': euiTheme.euiColorLightestShade,
+    },
+  };
+}
+
+export interface Props {
+  onChange: (scriptContent?: string) => void;
+  defaultValue?: string;
+}
+
+export default function WebScrapingResourcesTrackerScriptEditor({ onChange, defaultValue }: Props) {
+  const { colorMode } = useEuiTheme();
+
+  const monacoTheme = createTheme(
+    colorMode === 'DARK' ? v8Dark : v8Light,
+    colorMode === 'DARK' ? '#343551' : '#E3E4ED',
+  );
+
+  return (
+    <Editor
+      height="15vh"
+      defaultLanguage="javascript"
+      defaultValue={defaultValue}
+      onChange={(value) => onChange(value)}
+      theme={'euiTheme'}
+      beforeMount={(monaco) => {
+        monaco.editor.defineTheme('euiTheme', monacoTheme);
+      }}
+    />
+  );
+}
diff --git a/yarn.lock b/yarn.lock
@@ -333,6 +333,20 @@
     "@lezer/lr" "^0.15.4"
     json5 "^2.2.1"
 
+"@monaco-editor/loader@^1.3.3":
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/@monaco-editor/loader/-/loader-1.3.3.tgz#7f1742bd3cc21c0362a46a4056317f6e5215cfca"
+  integrity sha512-6KKF4CTzcJiS8BJwtxtfyYt9shBiEv32ateQ9T4UVogwn4HM/uPo9iJd2Dmbkpz8CM6Y0PDUpjnZzCwC+eYo2Q==
+  dependencies:
+    state-local "^1.0.6"
+
+"@monaco-editor/react@^4.5.2":
+  version "4.5.2"
+  resolved "https://registry.yarnpkg.com/@monaco-editor/react/-/react-4.5.2.tgz#e8cc802203f729b423a998ea6fcb466604d61258"
+  integrity sha512-emcWu6vg1OpXPiYll4aPOaXe8bwYB4UaaNTwtArFLgMoNGBzRZb2Xn0Bra2HMIFM7QLgs7fCGunHO5LkfT2LBA==
+  dependencies:
+    "@monaco-editor/loader" "^1.3.3"
+
 "@msgpackr-extract/[email protected]":
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz#44d752c1a2dc113f15f781b7cc4f53a307e3fa38"
@@ -3420,6 +3434,11 @@ moment@^2.29.4:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108"
   integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==
 
+monaco-editor@^0.41.0:
+  version "0.41.0"
+  resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.41.0.tgz#2ba31e5af7e3ae93ac5d7467ec2772ef9b3d967f"
+  integrity sha512-1o4olnZJsiLmv5pwLEAmzHTE/5geLKQ07BrGxlF4Ri/AXAc2yyDGZwHjiTqD8D/ROKUZmwMA28A+yEowLNOEcA==
+
 [email protected]:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
@@ -4307,6 +4326,11 @@ stable@^0.1.8:
   resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf"
   integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==
 
+state-local@^1.0.6:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/state-local/-/state-local-1.0.7.tgz#da50211d07f05748d53009bee46307a37db386d5"
+  integrity sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==
+
 state-toggle@^1.0.0:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe"