bfetch (2) (#53711)

* feat: 🎸 implement ItemBuffer

* test: 💍 add tests for ItemBuffer

* feat: 🎸 add TimedItemBuffer

* test: 💍 add TimedItemBuffer tests

* feat: 🎸 add createBatchedFunction

* chore: 🤖 save wip on higher level batching

* test: 💍 add createBatchedFunction tests

* feat: 🎸 implement createStreamingBatchedFunction() method

* refactor: 💡 rename "data" key to "result"

* feat: 🎸 return error in "error" key in legacy protocol

* feat: 🎸 add server-side to Expressions plugin

* refactor: 💡 move interpreter server-side registries to NP

* feat: 🎸 implement bfetch.addBatchProcessingRoute

* feat: 🎸 improve streaming and batching func to pass request

* feat: 🎸 initial setup of new expressions batching endpoint

* feat: 🎸 expose bfetch.batchedFunction() function

* feat: 🎸 add of() function

of() function awaits a promise and converts it to a 3-tuple representing
its state.

* refactor: 💡 move normalizeError() to /common

* feat: 🎸 improve createStreamingBatchedFunction() function

* refactor: 💡 move GET /api/interpreter/fns to the New Platform

* feat: 🎸 move batched_fetch to the New Platform

* feat: 🎸 implement legacy interpreter batching on server in NP

* feat: 🎸 switch legacy interpreter server functions to NP

* chore: 🤖 remove unused import

* fix: 🐛 correct expressions mocks

* test: 💍 fix batching tests after refactor

* test: 💍 stub bfetch plugin explorer

* test: 💍 add routing and app structure to bfetch_explorer

* test: 💍 add server-side to bfetch_explorer

* test: 💍 create <DoubleInteger> component in bfetch_explorer

* test: 💍 improve bfetch_explorer

* test: 💍 add <CountUntil> demo to bfetch_explorer

* test: 💍 by default redirect to first bfetch_explorer example

* test: 💍 add error example to bfetch_explorer

* docs: ✏️ improve bfetch docs

* docs: ✏️ improve bfetch server-side docs

* chore: 🤖 address self-review comments

* fix: 🐛 use new core ES data client, remove unuseed import

* fix: 🐛 remove unused interface import

* Update examples/bfetch_explorer/public/components/count_until/index.tsx

Co-Authored-By: Lukas Olson <[email protected]>

* Update examples/bfetch_explorer/public/components/double_integers/index.tsx

Co-Authored-By: Lukas Olson <[email protected]>

* Update src/plugins/bfetch/common/buffer/item_buffer.ts

Co-Authored-By: Lukas Olson <[email protected]>

* Update src/plugins/kibana_utils/common/of.ts

Co-Authored-By: Lukas Olson <[email protected]>

* docs: ✏️ add batchedFunction params to README

* refactor: 💡 rename onflush to onFlush

* feat: 🎸 make maxItemAge optional in TimedItemBuffer

* refactor: 💡 remove promise from fetchStreaming

* test: 💍 add bfetch_explorer functional tests

* test: 💍 rename test plugin to kbn_tp_bfetch_explorer

* fix: 🐛 use stream instead of removed promise

* fix: 🐛 use correct tsconfig.json in bfetch test plugin

* feat: 🎸 add kbn_tp_bfetch_explorer server-side files to tsconfi

Co-authored-by: Lukas Olson <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>

examples/README.md

@@ -5,4 +5,3 @@ This folder contains example plugins.  To run the plugins in this folder, use th
 ```
 yarn start --run-examples
 ```
-

examples/bfetch_explorer/kibana.json

@@ -0,0 +1,10 @@
+{
+  "id": "bfetchExplorer",
+  "version": "0.0.1",
+  "kibanaVersion": "kibana",
+  "configPath": ["bfetch_explorer"],
+  "server": true,
+  "ui": true,
+  "requiredPlugins": ["bfetch"],
+  "optionalPlugins": []
+}

examples/bfetch_explorer/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "bfetch_explorer",
+  "version": "1.0.0",
+  "main": "target/examples/bfetch_explorer",
+  "kibana": {
+    "version": "kibana",
+    "templateVersion": "1.0.0"
+  },
+  "license": "Apache-2.0",
+  "scripts": {
+    "kbn": "node ../../scripts/kbn.js",
+    "build": "rm -rf './target' && tsc"
+  },
+  "devDependencies": {
+    "typescript": "3.7.2"
+  }
+}

examples/bfetch_explorer/public/components/count_until/index.tsx

@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useState } from 'react';
+import useMountedState from 'react-use/lib/useMountedState';
+import useList from 'react-use/lib/useList';
+import { EuiForm, EuiSpacer, EuiFieldNumber, EuiFormRow, EuiButton } from '@elastic/eui';
+import { BfetchPublicSetup } from '../../../../../src/plugins/bfetch/public';
+
+export interface Props {
+  fetchStreaming: BfetchPublicSetup['fetchStreaming'];
+}
+
+export const CountUntil: React.FC<Props> = ({ fetchStreaming }) => {
+  const isMounted = useMountedState();
+  const [data, setData] = useState(5);
+  const [showingResults, setShowingResults] = useState(false);
+  const [results, { push: pushResult, clear: clearList }] = useList<string>([]);
+  const [completed, setCompleted] = useState(false);
+  const [error, setError] = useState<any>(null);
+
+  const handleSubmit = () => {
+    setShowingResults(true);
+    const { stream } = fetchStreaming({
+      url: '/bfetch_explorer/count',
+      body: JSON.stringify({ data }),
+    });
+    stream.subscribe({
+      next: (next: string) => {
+        if (!isMounted()) return;
+        pushResult(next);
+      },
+      error: (nextError: any) => {
+        if (!isMounted()) return;
+        setError(nextError);
+      },
+      complete: () => {
+        if (!isMounted()) return;
+        setCompleted(true);
+      },
+    });
+  };
+
+  const handleReset = () => {
+    setShowingResults(false);
+    clearList();
+    setError(null);
+    setCompleted(false);
+  };
+
+  if (showingResults) {
+    return (
+      <EuiForm data-test-subj="CountUntil">
+        <pre>{JSON.stringify(error || results, null, 4)}</pre>
+        <EuiSpacer size="l" />
+        <EuiButton disabled={!completed} onClick={handleReset}>
+          Reset
+        </EuiButton>
+      </EuiForm>
+    );
+  }
+
+  return (
+    <EuiForm data-test-subj="CountUntil">
+      <EuiFormRow label="Some integer" fullWidth>
+        <EuiFieldNumber
+          placeholder="Some integer"
+          value={data}
+          onChange={e => setData(Number(e.target.value))}
+        />
+      </EuiFormRow>
+      <EuiButton type="submit" fill onClick={handleSubmit}>
+        Start
+      </EuiButton>
+    </EuiForm>
+  );
+};

examples/bfetch_explorer/public/components/double_integers/index.tsx

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useState } from 'react';
+import useMountedState from 'react-use/lib/useMountedState';
+import useList from 'react-use/lib/useList';
+import useCounter from 'react-use/lib/useCounter';
+import { EuiForm, EuiSpacer, EuiTextArea, EuiFormRow, EuiButton } from '@elastic/eui';
+import { ExplorerService } from '../../plugin';
+
+interface ResultItem {
+  num: number;
+  result?: {
+    num: number;
+  };
+  error?: any;
+}
+
+const defaultNumbers = [2000, 300, -1, 1000].join('\n');
+
+export interface Props {
+  double: ExplorerService['double'];
+}
+
+export const DoubleIntegers: React.FC<Props> = ({ double }) => {
+  const isMounted = useMountedState();
+  const [numbers, setNumbers] = useState(defaultNumbers);
+  const [showingResults, setShowingResults] = useState(false);
+  const [numberOfResultsAwaiting, counter] = useCounter(0);
+  const [results, { push: pushResult, clear: clearList }] = useList<ResultItem>([]);
+
+  const handleSubmit = () => {
+    setShowingResults(true);
+    const nums = numbers
+      .split('\n')
+      .map(num => num.trim())
+      .filter(Boolean)
+      .map(Number);
+    counter.set(nums.length);
+    nums.forEach(num => {
+      double({ num }).then(
+        result => {
+          if (!isMounted()) return;
+          counter.dec();
+          pushResult({ num, result });
+        },
+        error => {
+          if (!isMounted()) return;
+          counter.dec();
+          pushResult({ num, error });
+        }
+      );
+    });
+  };
+
+  const handleReset = () => {
+    setShowingResults(false);
+    counter.reset();
+    clearList();
+  };
+
+  if (showingResults) {
+    return (
+      <EuiForm data-test-subj="DoubleIntegers">
+        <pre>{JSON.stringify(results, null, 4)}</pre>
+        <EuiSpacer size="l" />
+        <EuiButton disabled={!!numberOfResultsAwaiting} onClick={handleReset}>
+          Reset
+        </EuiButton>
+      </EuiForm>
+    );
+  }
+
+  return (
+    <EuiForm data-test-subj="DoubleIntegers">
+      <EuiFormRow label="Numbers in ms separated by new line" fullWidth>
+        <EuiTextArea
+          fullWidth
+          placeholder="Enter numbers in milliseconds separated by new line"
+          value={numbers}
+          onChange={e => setNumbers(e.target.value)}
+        />
+      </EuiFormRow>
+      <EuiButton type="submit" fill onClick={handleSubmit}>
+        Send
+      </EuiButton>
+    </EuiForm>
+  );
+};

examples/bfetch_explorer/public/components/page/index.tsx

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import * as React from 'react';
+import {
+  EuiPageBody,
+  EuiPageContent,
+  EuiPageContentBody,
+  EuiPageHeader,
+  EuiPageHeaderSection,
+  EuiTitle,
+} from '@elastic/eui';
+
+export interface PageProps {
+  title?: React.ReactNode;
+}
+
+export const Page: React.FC<PageProps> = ({ title = 'Untitled', children }) => {
+  return (
+    <EuiPageBody>
+      <EuiPageHeader>
+        <EuiPageHeaderSection>
+          <EuiTitle size="l">
+            <h1>{title}</h1>
+          </EuiTitle>
+        </EuiPageHeaderSection>
+      </EuiPageHeader>
+      <EuiPageContent>
+        <EuiPageContentBody style={{ maxWidth: 800, margin: '0 auto' }}>
+          {children}
+        </EuiPageContentBody>
+      </EuiPageContent>
+    </EuiPageBody>
+  );
+};

examples/bfetch_explorer/public/containers/app/index.tsx

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from 'react';
+import { BrowserRouter as Router, Route, Redirect, Switch } from 'react-router-dom';
+import { EuiPage } from '@elastic/eui';
+import { useDeps } from '../../hooks/use_deps';
+import { Sidebar } from './sidebar';
+import { routes } from '../../routes';
+
+export const App: React.FC = () => {
+  const { appBasePath } = useDeps();
+
+  const routeElements: React.ReactElement[] = [];
+  for (const { items } of routes) {
+    for (const { id, component } of items) {
+      routeElements.push(<Route key={id} path={`/${id}`} render={props => component} />);
+    }
+  }
+
+  return (
+    <Router basename={appBasePath}>
+      <EuiPage>
+        <Sidebar />
+        <Switch>
+          {routeElements}
+          <Redirect to="/count-until" />
+        </Switch>
+      </EuiPage>
+    </Router>
+  );
+};

examples/bfetch_explorer/public/containers/app/pages/page_count_until/index.tsx

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import * as React from 'react';
+import { EuiPanel, EuiText } from '@elastic/eui';
+import { CountUntil } from '../../../../components/count_until';
+import { Page } from '../../../../components/page';
+import { useDeps } from '../../../../hooks/use_deps';
+
+// eslint-disable-next-line
+export interface Props {}
+
+export const PageCountUntil: React.FC<Props> = () => {
+  const { plugins } = useDeps();
+
+  return (
+    <Page title={'Count Until'}>
+      <EuiText>
+        This demo sends a single number N using <code>fetchStreaming</code> to the server. The
+        server will stream back N number of messages with 1 second delay each containing a number
+        from 1 to N, after which it will close the stream.
+      </EuiText>
+      <br />
+      <EuiPanel paddingSize="l">
+        <CountUntil fetchStreaming={plugins.bfetch.fetchStreaming} />
+      </EuiPanel>
+    </Page>
+  );
+};