Merge remote-tracking branch 'upstream/main' into obs-rac-layout-cleanup

.buildkite/pipelines/es_snapshots/verify.yml

@@ -27,9 +27,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -41,7 +41,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -77,7 +77,7 @@ steps:
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: jest
+      queue: n2-2
     timeout_in_minutes: 120
     key: api-integration
 

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -8,7 +8,7 @@ const stepInput = (key, nameOfSuite) => {
 };
 
 const OSS_CI_GROUPS = 12;
-const XPACK_CI_GROUPS = 13;
+const XPACK_CI_GROUPS = 27;
 
 const inputs = [
   {

.buildkite/pipelines/flaky_tests/runner.js

@@ -78,7 +78,7 @@ for (const testSuite of testSuites) {
         steps.push({
           command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
           label: `Default CI Group ${CI_GROUP}`,
-          agents: { queue: 'ci-group-6' },
+          agents: { queue: 'n2-4' },
           depends_on: 'build',
           parallelism: RUN_COUNT,
           concurrency: concurrency,
@@ -103,7 +103,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${IS_XPACK ? 'xpack' : 'oss'}_firefox.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Firefox`,
-        agents: { queue: IS_XPACK ? 'ci-group-6' : 'ci-group-4d' },
+        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
         depends_on: 'build',
         parallelism: RUN_COUNT,
         concurrency: concurrency,
@@ -118,7 +118,7 @@ for (const testSuite of testSuites) {
           IS_XPACK ? 'xpack' : 'oss'
         }_accessibility.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Accessibility`,
-        agents: { queue: IS_XPACK ? 'ci-group-6' : 'ci-group-4d' },
+        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
         depends_on: 'build',
         parallelism: RUN_COUNT,
         concurrency: concurrency,

.buildkite/pipelines/hourly.yml

@@ -17,9 +17,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 250
     key: default-cigroup
@@ -31,7 +31,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -67,7 +67,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -89,7 +89,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -100,7 +100,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -111,7 +111,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:

.buildkite/pipelines/pull_request/base.yml

@@ -15,9 +15,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -29,7 +29,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -65,7 +65,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -87,7 +87,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -98,7 +98,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -109,7 +109,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -156,7 +156,7 @@ steps:
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: c2-4
+      queue: c2-8
     key: checks
     timeout_in_minutes: 120
 

.buildkite/scripts/steps/checks/type_check_plugin_public_api_docs.sh

@@ -11,9 +11,27 @@ checks-reporter-with-killswitch "Build TS Refs" \
     --no-cache \
     --force
 
-echo --- Check Types
 checks-reporter-with-killswitch "Check Types" \
-  node scripts/type_check
+  node scripts/type_check &> target/check_types.log &
+check_types_pid=$!
+
+node --max-old-space-size=12000 scripts/build_api_docs &> target/build_api_docs.log &
+api_docs_pid=$!
+
+wait $check_types_pid
+check_types_exit=$?
+
+wait $api_docs_pid
+api_docs_exit=$?
+
+echo --- Check Types
+cat target/check_types.log
+if [[ "$check_types_exit" != "0" ]]; then echo "^^^ +++"; fi
 
 echo --- Building api docs
-node --max-old-space-size=12000 scripts/build_api_docs
+cat target/build_api_docs.log
+if [[ "$api_docs_exit" != "0" ]]; then echo "^^^ +++"; fi
+
+if [[ "${api_docs_exit}${check_types_exit}" != "00" ]]; then
+  exit 1
+fi

.ci/ci_groups.yml

@@ -25,4 +25,18 @@ xpack:
   - ciGroup11
   - ciGroup12
   - ciGroup13
+  - ciGroup14
+  - ciGroup15
+  - ciGroup16
+  - ciGroup17
+  - ciGroup18
+  - ciGroup19
+  - ciGroup20
+  - ciGroup21
+  - ciGroup22
+  - ciGroup23
+  - ciGroup24
+  - ciGroup25
+  - ciGroup26
+  - ciGroup27
   - ciGroupDocker

docs/settings/security-settings.asciidoc

@@ -332,7 +332,7 @@ For more details and a reference of audit events, refer to <<xpack-security-audi
 xpack.security.audit.enabled: true
 xpack.security.audit.appender: <1>
   type: rolling-file
-  fileName: ./data/audit.log
+  fileName: ./logs/audit.log
   policy:
     type: time-interval
     interval: 24h <2>

package.json

@@ -761,7 +761,7 @@
     "mocha-junit-reporter": "^2.0.0",
     "mochawesome": "^6.2.1",
     "mochawesome-merge": "^4.2.0",
-    "mock-fs": "^5.1.1",
+    "mock-fs": "^5.1.2",
     "mock-http-server": "1.3.0",
     "ms-chromium-edge-driver": "^0.4.2",
     "multimatch": "^4.0.0",

packages/kbn-cli-dev-mode/src/dev_server.test.ts

@@ -80,6 +80,7 @@ beforeEach(() => {
   jest.clearAllMocks();
   log.messages.length = 0;
   process.execArgv = ['--inheritted', '--exec', '--argv'];
+  process.env.FORCE_COLOR = process.env.FORCE_COLOR || '1';
   currentProc = undefined;
 });
 
@@ -117,13 +118,10 @@ afterEach(() => {
   subscriptions.length = 0;
 });
 
-describe.skip('#run$', () => {
+describe('#run$', () => {
   it('starts the dev server with the right options', () => {
     run(new DevServer(defaultOptions)).unsubscribe();
 
-    // ensure that FORCE_COLOR is in the env for consistency in snapshot
-    process.env.FORCE_COLOR = process.env.FORCE_COLOR || 'true';
-
     expect(execa.node.mock.calls).toMatchInlineSnapshot(`
       Array [
         Array [
@@ -136,6 +134,7 @@ describe.skip('#run$', () => {
             "env": Object {
               "<inheritted process.env>": true,
               "ELASTIC_APM_SERVICE_NAME": "kibana",
+              "FORCE_COLOR": "true",
               "isDevCliChild": "true",
             },
             "nodeOptions": Array [

packages/kbn-utils/src/path/index.test.ts

@@ -7,21 +7,35 @@
  */
 
 import { accessSync, constants } from 'fs';
-import { getConfigPath, getDataPath, getConfigDirectory } from './';
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+import { getConfigPath, getDataPath, getLogsPath, getConfigDirectory } from './';
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer());
 
 describe('Default path finder', () => {
-  it('should find a kibana.yml', () => {
-    const configPath = getConfigPath();
-    expect(() => accessSync(configPath, constants.R_OK)).not.toThrow();
+  it('should expose a path to the config directory', () => {
+    expect(getConfigDirectory()).toMatchInlineSnapshot('<absolute path>/config');
   });
 
-  it('should find a data directory', () => {
-    const dataPath = getDataPath();
-    expect(() => accessSync(dataPath, constants.R_OK)).not.toThrow();
+  it('should expose a path to the kibana.yml', () => {
+    expect(getConfigPath()).toMatchInlineSnapshot('<absolute path>/config/kibana.yml');
+  });
+
+  it('should expose a path to the data directory', () => {
+    expect(getDataPath()).toMatchInlineSnapshot('<absolute path>/data');
+  });
+
+  it('should expose a path to the logs directory', () => {
+    expect(getLogsPath()).toMatchInlineSnapshot('<absolute path>/logs');
   });
 
   it('should find a config directory', () => {
     const configDirectory = getConfigDirectory();
     expect(() => accessSync(configDirectory, constants.R_OK)).not.toThrow();
   });
+
+  it('should find a kibana.yml', () => {
+    const configPath = getConfigPath();
+    expect(() => accessSync(configPath, constants.R_OK)).not.toThrow();
+  });
 });

packages/kbn-utils/src/path/index.ts

@@ -27,6 +27,8 @@ const CONFIG_DIRECTORIES = [
 
 const DATA_PATHS = [join(REPO_ROOT, 'data'), '/var/lib/kibana'].filter(isString);
 
+const LOGS_PATHS = [join(REPO_ROOT, 'logs'), '/var/log/kibana'].filter(isString);
+
 function findFile(paths: string[]) {
   const availablePath = paths.find((configPath) => {
     try {
@@ -57,6 +59,12 @@ export const getConfigDirectory = () => findFile(CONFIG_DIRECTORIES);
  */
 export const getDataPath = () => findFile(DATA_PATHS);
 
+/**
+ * Get the directory containing logs
+ * @internal
+ */
+export const getLogsPath = () => findFile(LOGS_PATHS);
+
 export type PathConfigType = TypeOf<typeof config.schema>;
 
 export const config = {