[6.x] [retry] implement waitFor method (#21747) (#21969)

Backports the following commits to 6.x:
 - [retry] implement waitFor method  (#21747)

src/functional_test_runner/lib/config/schema.js

@@ -73,6 +73,7 @@ export const schema = Joi.object().keys({
   timeouts: Joi.object().keys({
     find: Joi.number().default(10000),
     try: Joi.number().default(40000),
+    waitFor: Joi.number().default(20000),
     esRequestTimeout: Joi.number().default(30000),
     kibanaStabilize: Joi.number().default(15000),
     navigateStatusPageCheck: Joi.number().default(250),

test/common/services/retry/index.js

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { RetryProvider } from './retry';

test/common/services/retry/retry.js

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { retryForTruthy } from './retry_for_truthy';
+import { retryForSuccess } from './retry_for_success';
+
+export function RetryProvider({ getService }) {
+  const config = getService('config');
+  const log = getService('log');
+
+  return new class Retry {
+    async tryForTime(timeout, block) {
+      return await retryForSuccess(log, {
+        timeout,
+        methodName: 'retry.tryForTime',
+        block
+      });
+    }
+
+    async try(block) {
+      return await retryForSuccess(log, {
+        timeout: config.get('timeouts.try'),
+        methodName: 'retry.try',
+        block
+      });
+    }
+
+    async tryMethod(object, method, ...args) {
+      return await retryForSuccess(log, {
+        timeout: config.get('timeouts.try'),
+        methodName: 'retry.tryMethod',
+        block: async () => (
+          await object[method](...args)
+        )
+      });
+    }
+
+    async waitForWithTimeout(description, timeout, block) {
+      await retryForTruthy(log, {
+        timeout,
+        methodName: 'retry.waitForWithTimeout',
+        description,
+        block
+      });
+    }
+
+    async waitFor(description, block) {
+      await retryForTruthy(log, {
+        timeout: config.get('timeouts.waitFor'),
+        methodName: 'retry.waitFor',
+        description,
+        block
+      });
+    }
+  };
+}

test/common/services/retry/retry_for_success.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { inspect } from 'util';
+
+const delay = ms => new Promise(resolve => (
+  setTimeout(resolve, ms)
+));
+
+const returnTrue = () => true;
+
+const defaultOnFailure = (methodName) => (lastError) => {
+  throw new Error(`${methodName} timeout: ${lastError.stack || lastError.message}`);
+};
+
+/**
+ * Run a function and return either an error or result
+ * @param {Function} block
+ */
+async function runAttempt(block) {
+  try {
+    return {
+      result: await block()
+    };
+  } catch (error) {
+    return {
+      // we rely on error being truthy and throwing falsy values is *allowed*
+      // so we cast falsy values to errors
+      error: error || new Error(`${inspect(error)} thrown`),
+    };
+  }
+}
+
+export async function retryForSuccess(log, {
+  timeout,
+  methodName,
+  block,
+  onFailure = defaultOnFailure(methodName),
+  accept = returnTrue
+}) {
+  const start = Date.now();
+  const retryDelay = 502;
+  let lastError;
+
+  while (true) {
+    if (Date.now() - start > timeout) {
+      await onFailure(lastError);
+      throw new Error('expected onFailure() option to throw an error');
+    }
+
+    const { result, error } = await runAttempt(block);
+
+    if (!error && accept(result)) {
+      return result;
+    }
+
+    if (error) {
+      if (lastError && lastError.message === error.message) {
+        log.debug(`--- ${methodName} failed again with the same message...`);
+      } else {
+        log.debug(`--- ${methodName} error: ${error.message}`);
+      }
+
+      lastError = error;
+    }
+
+    await delay(retryDelay);
+  }
+}

test/common/services/retry/retry_for_truthy.js

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { retryForSuccess } from './retry_for_success';
+
+export async function retryForTruthy(log, {
+  timeout,
+  methodName,
+  description,
+  block
+}) {
+  log.debug(`Waiting up to ${timeout}ms for ${description}...`);
+
+  const accept = result => Boolean(result);
+
+  const onFailure = lastError => {
+    let msg = `timed out waiting for ${description}`;
+
+    if (lastError) {
+      msg = `${msg} -- last error: ${lastError.stack || lastError.message}`;
+    }
+
+    throw new Error(msg);
+  };
+
+  await retryForSuccess(log, {
+    timeout,
+    methodName,
+    block,
+    onFailure,
+    accept
+  });
+}