[Synthetics] Make core API key include read_ilm privilege in Statef…

…ul only (#178897)

## Summary

Resolves elastic/synthetics-dev#332.

ILM is not a concept in Serverless. As such, when the Synthetics plugin
requests the `read_ilm` permission for its core API key during
bootstrapping, it's asking for a priv that will eventually not exist on
Serverless, and ES will give an explicit deny for the request, which
will probably cause Synthetics to crash and not be functional.

The fix is to detect the build type at startup time and enable the
server plugin to determine whether it needs to include this privilege or
not, based on whether Kibana is running in stateful or serverless mode.

## Testing

You can easily test this in both modes. The steps are the same.

_NOTE:_ when testing serverless, if you include the flag ` -E
xpack.security.authz.has_privileges.strict_request_validation.enabled=true`
this will simulate the manner in which Elasticsearch will explicit deny
the API creation request when this is enabled in the MKI environment,
and thus you should include it in your testing.

1. Start up Kibana in serverless | stateful mode.
2. As an admin, navigate to Synthetics and wait for the startup flow to
display.
3. Navigate to Kibana management's API key page. You should see the
Synthetics API key. Click it.
4. For stateful, you should see `read_ilm` included under the
`synthetics_writer` object, shown below. For serverless, you should not
see this priv in the list.

### Stateful API key perms

<img width="1834" alt="image"
src="https://github.com/elastic/kibana/assets/18429259/09048a7d-dcea-420e-bef5-87f86e447791">

### Serverless API key perms

<img width="1844" alt="image"
src="https://github.com/elastic/kibana/assets/18429259/9a2a7f6b-6c6a-42f9-a47a-2ee157b2692b">

x-pack/plugins/observability_solution/synthetics/server/plugin.ts

@@ -34,16 +34,14 @@ import { uptimeRuleTypeFieldMap } from './alert_rules/common';
 
 export class Plugin implements PluginType {
   private savedObjectsClient?: SavedObjectsClientContract;
-  private initContext: PluginInitializerContext;
-  private logger: Logger;
+  private readonly logger: Logger;
   private server?: SyntheticsServerSetup;
   private syntheticsService?: SyntheticsService;
   private syntheticsMonitorClient?: SyntheticsMonitorClient;
   private readonly telemetryEventsSender: TelemetryEventsSender;
 
-  constructor(initializerContext: PluginInitializerContext<UptimeConfig>) {
-    this.initContext = initializerContext;
-    this.logger = initializerContext.logger.get();
+  constructor(private readonly initContext: PluginInitializerContext<UptimeConfig>) {
+    this.logger = initContext.logger.get();
     this.telemetryEventsSender = new TelemetryEventsSender(this.logger);
   }
 
@@ -52,7 +50,6 @@ export class Plugin implements PluginType {
 
     savedObjectsAdapter.config = config;
 
-    this.logger = this.initContext.logger.get();
     const { ruleDataService } = plugins.ruleRegistry;
 
     const ruleDataClient = ruleDataService.initializeIndex({
@@ -110,6 +107,7 @@ export class Plugin implements PluginType {
       this.server.encryptedSavedObjects = pluginsStart.encryptedSavedObjects;
       this.server.savedObjectsClient = this.savedObjectsClient;
       this.server.spaces = pluginsStart.spaces;
+      this.server.isElasticsearchServerless = coreStart.elasticsearch.getCapabilities().serverless;
     }
 
     this.syntheticsService?.start(pluginsStart.taskManager);

x-pack/plugins/observability_solution/synthetics/server/synthetics_service/authentication/check_has_privilege.ts

@@ -9,24 +9,25 @@ import { SecurityIndexPrivilege } from '@elastic/elasticsearch/lib/api/types';
 import { UptimeEsClient } from '../../lib';
 import { SyntheticsServerSetup } from '../../types';
 import { getFakeKibanaRequest } from '../utils/fake_kibana_request';
-import { serviceApiKeyPrivileges, syntheticsIndex } from '../get_api_key';
+import { getServiceApiKeyPrivileges, syntheticsIndex } from '../get_api_key';
 
-export const checkHasPrivileges = async (
+export const checkHasPrivileges = (
   server: SyntheticsServerSetup,
   apiKey: { id: string; apiKey: string }
 ) => {
-  return await server.coreStart.elasticsearch.client
+  const { indices: index, cluster } = getServiceApiKeyPrivileges(server.isElasticsearchServerless);
+  return server.coreStart.elasticsearch.client
     .asScoped(getFakeKibanaRequest({ id: apiKey.id, api_key: apiKey.apiKey }))
     .asCurrentUser.security.hasPrivileges({
       body: {
-        index: serviceApiKeyPrivileges.indices,
-        cluster: serviceApiKeyPrivileges.cluster,
+        index,
+        cluster,
       },
     });
 };
 
-export const checkIndicesReadPrivileges = async (uptimeEsClient: UptimeEsClient) => {
-  return await uptimeEsClient.baseESClient.security.hasPrivileges({
+export const checkIndicesReadPrivileges = (uptimeEsClient: UptimeEsClient) => {
+  return uptimeEsClient.baseESClient.security.hasPrivileges({
     body: {
       index: [
         {

x-pack/plugins/observability_solution/synthetics/server/synthetics_service/get_api_key.test.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { getAPIKeyForSyntheticsService, syntheticsIndex } from './get_api_key';
+import {
+  getAPIKeyForSyntheticsService,
+  getServiceApiKeyPrivileges,
+  syntheticsIndex,
+} from './get_api_key';
 import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks';
 import { securityMock } from '@kbn/security-plugin/server/mocks';
 import { coreMock } from '@kbn/core/server/mocks';
@@ -84,6 +88,18 @@ describe('getAPIKeyTest', function () {
     );
   });
 
+  it.each([
+    [true, ['monitor', 'read_pipeline']],
+    [false, ['monitor', 'read_pipeline', 'read_ilm']],
+  ])(
+    'Includes/excludes `read_ilm` priv when serverless is mode is %s',
+    (isServerlessEs, expectedClusterPrivs) => {
+      const { cluster } = getServiceApiKeyPrivileges(isServerlessEs);
+
+      expect(cluster).toEqual(expectedClusterPrivs);
+    }
+  );
+
   it('invalidates api keys with missing read permissions', async () => {
     jest.spyOn(authUtils, 'checkHasPrivileges').mockResolvedValue({
       index: {

x-pack/plugins/observability_solution/synthetics/server/synthetics_service/get_api_key.ts

@@ -19,20 +19,24 @@ import { checkHasPrivileges } from './authentication/check_has_privilege';
 
 export const syntheticsIndex = 'synthetics-*';
 
-export const serviceApiKeyPrivileges = {
-  cluster: ['monitor', 'read_ilm', 'read_pipeline'] as SecurityClusterPrivilege[],
-  indices: [
-    {
-      names: [syntheticsIndex],
-      privileges: [
-        'view_index_metadata',
-        'create_doc',
-        'auto_configure',
-        'read',
-      ] as SecurityIndexPrivilege[],
-    },
-  ],
-  run_as: [],
+export const getServiceApiKeyPrivileges = (isServerlessEs: boolean) => {
+  const cluster: SecurityClusterPrivilege[] = ['monitor', 'read_pipeline'];
+  if (isServerlessEs === false) cluster.push('read_ilm');
+  return {
+    cluster,
+    indices: [
+      {
+        names: [syntheticsIndex],
+        privileges: [
+          'view_index_metadata',
+          'create_doc',
+          'auto_configure',
+          'read',
+        ] as SecurityIndexPrivilege[],
+      },
+    ],
+    run_as: [],
+  };
 };
 
 export const getAPIKeyForSyntheticsService = async ({
@@ -84,7 +88,7 @@ export const generateAPIKey = async ({
   server: SyntheticsServerSetup;
   request: KibanaRequest;
 }) => {
-  const { security } = server;
+  const { isElasticsearchServerless, security } = server;
   const isApiKeysEnabled = await security.authc.apiKeys?.areAPIKeysEnabled();
 
   if (!isApiKeysEnabled) {
@@ -100,7 +104,7 @@ export const generateAPIKey = async ({
   return security.authc.apiKeys?.grantAsInternalUser(request, {
     name: 'synthetics-api-key (required for Synthetics App)',
     role_descriptors: {
-      synthetics_writer: serviceApiKeyPrivileges,
+      synthetics_writer: getServiceApiKeyPrivileges(isElasticsearchServerless),
     },
     metadata: {
       description:
@@ -202,16 +206,16 @@ export const getSyntheticsEnablement = async ({ server }: { server: SyntheticsSe
   };
 };
 
-const hasEnablePermissions = async ({ uptimeEsClient }: SyntheticsServerSetup) => {
+const hasEnablePermissions = async ({
+  uptimeEsClient,
+  isElasticsearchServerless,
+}: SyntheticsServerSetup) => {
+  const { cluster: clusterPrivs, indices: index } =
+    getServiceApiKeyPrivileges(isElasticsearchServerless);
   const hasPrivileges = await uptimeEsClient.baseESClient.security.hasPrivileges({
     body: {
-      cluster: [
-        'manage_security',
-        'manage_api_key',
-        'manage_own_api_key',
-        ...serviceApiKeyPrivileges.cluster,
-      ],
-      index: serviceApiKeyPrivileges.indices,
+      cluster: ['manage_security', 'manage_api_key', 'manage_own_api_key', ...clusterPrivs],
+      index,
     },
   });
 
@@ -221,7 +225,9 @@ const hasEnablePermissions = async ({ uptimeEsClient }: SyntheticsServerSetup) =
     manage_api_key: manageApiKey,
     manage_own_api_key: manageOwnApiKey,
     monitor,
-    read_ilm: readILM,
+    // `read_ilm` is going to be `undefined` when ES is in serverless mode,
+    // so we default it to the ES capabilities value.
+    read_ilm: readILM = isElasticsearchServerless,
     read_pipeline: readPipeline,
   } = cluster || {};
 

x-pack/plugins/observability_solution/synthetics/server/types.ts

@@ -59,6 +59,7 @@ export interface SyntheticsServerSetup {
   isDev?: boolean;
   coreStart: CoreStart;
   pluginsStart: SyntheticsPluginsStartDependencies;
+  isElasticsearchServerless: boolean;
 }
 
 export interface SyntheticsPluginsSetupDependencies {

x-pack/test/api_integration/apis/security/api_keys.ts

@@ -7,7 +7,7 @@
 
 import expect from '@kbn/expect';
 import { ALL_SPACES_ID } from '@kbn/security-plugin/common/constants';
-import { serviceApiKeyPrivileges } from '@kbn/synthetics-plugin/server/synthetics_service/get_api_key';
+import { getServiceApiKeyPrivileges } from '@kbn/synthetics-plugin/server/synthetics_service/get_api_key';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService }: FtrProviderContext) {
@@ -116,7 +116,7 @@ export default function ({ getService }: FtrProviderContext) {
               expiration: '12d',
               kibana_role_descriptors: {
                 uptime_save: {
-                  elasticsearch: serviceApiKeyPrivileges,
+                  elasticsearch: getServiceApiKeyPrivileges(false),
                   kibana: [
                     {
                       base: [],

x-pack/test/api_integration/apis/synthetics/add_monitor.ts

@@ -21,7 +21,7 @@ import { ALL_SPACES_ID } from '@kbn/security-plugin/common/constants';
 import { format as formatUrl } from 'url';
 
 import supertest from 'supertest';
-import { serviceApiKeyPrivileges } from '@kbn/synthetics-plugin/server/synthetics_service/get_api_key';
+import { getServiceApiKeyPrivileges } from '@kbn/synthetics-plugin/server/synthetics_service/get_api_key';
 import { syntheticsMonitorType } from '@kbn/synthetics-plugin/common/types/saved_objects';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { getFixtureJson } from './helper/get_fixture_json';
@@ -217,7 +217,7 @@ export default function ({ getService }: FtrProviderContext) {
           expiration: '12d',
           kibana_role_descriptors: {
             uptime_save: {
-              elasticsearch: serviceApiKeyPrivileges,
+              elasticsearch: getServiceApiKeyPrivileges(false),
               kibana: [
                 {
                   base: [],
@@ -260,7 +260,7 @@ export default function ({ getService }: FtrProviderContext) {
           expiration: '12d',
           kibana_role_descriptors: {
             uptime_save: {
-              elasticsearch: serviceApiKeyPrivileges,
+              elasticsearch: getServiceApiKeyPrivileges(false),
               kibana: [
                 {
                   base: [],

x-pack/test/api_integration/apis/synthetics/synthetics_enablement.ts

@@ -10,14 +10,14 @@ import {
   syntheticsApiKeyID,
   syntheticsApiKeyObjectType,
 } from '@kbn/synthetics-plugin/server/saved_objects/service_api_key';
-import { serviceApiKeyPrivileges } from '@kbn/synthetics-plugin/server/synthetics_service/get_api_key';
+import { getServiceApiKeyPrivileges } from '@kbn/synthetics-plugin/server/synthetics_service/get_api_key';
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService }: FtrProviderContext) {
   const correctPrivileges = {
     applications: [],
-    cluster: ['monitor', 'read_ilm', 'read_pipeline'],
+    cluster: ['monitor', 'read_pipeline', 'read_ilm'],
     indices: [
       {
         allow_restricted_indices: false,
@@ -74,7 +74,7 @@ export default function ({ getService }: FtrProviderContext) {
               ],
               elasticsearch: {
                 cluster: [privilege],
-                indices: serviceApiKeyPrivileges.indices,
+                indices: getServiceApiKeyPrivileges(false).indices,
               },
             });
 
@@ -119,8 +119,8 @@ export default function ({ getService }: FtrProviderContext) {
               },
             ],
             elasticsearch: {
-              cluster: serviceApiKeyPrivileges.cluster,
-              indices: serviceApiKeyPrivileges.indices,
+              cluster: getServiceApiKeyPrivileges(false).cluster,
+              indices: getServiceApiKeyPrivileges(false).indices,
             },
           });
 
@@ -167,8 +167,8 @@ export default function ({ getService }: FtrProviderContext) {
               },
             ],
             elasticsearch: {
-              cluster: serviceApiKeyPrivileges.cluster,
-              indices: serviceApiKeyPrivileges.indices,
+              cluster: getServiceApiKeyPrivileges(false).cluster,
+              indices: getServiceApiKeyPrivileges(false).indices,
             },
           });
 
@@ -233,7 +233,7 @@ export default function ({ getService }: FtrProviderContext) {
               expiration: '1d',
               role_descriptors: {
                 'role-a': {
-                  cluster: serviceApiKeyPrivileges.cluster,
+                  cluster: getServiceApiKeyPrivileges(false).cluster,
                   indices: [
                     {
                       names: ['synthetics-*'],
@@ -269,8 +269,8 @@ export default function ({ getService }: FtrProviderContext) {
               },
             ],
             elasticsearch: {
-              cluster: serviceApiKeyPrivileges.cluster,
-              indices: serviceApiKeyPrivileges.indices,
+              cluster: getServiceApiKeyPrivileges(false).cluster,
+              indices: getServiceApiKeyPrivileges(false).indices,
             },
           });
 
@@ -318,8 +318,8 @@ export default function ({ getService }: FtrProviderContext) {
               },
             ],
             elasticsearch: {
-              cluster: serviceApiKeyPrivileges.cluster,
-              indices: serviceApiKeyPrivileges.indices,
+              cluster: getServiceApiKeyPrivileges(false).cluster,
+              indices: getServiceApiKeyPrivileges(false).indices,
             },
           });
 
@@ -449,8 +449,8 @@ export default function ({ getService }: FtrProviderContext) {
               },
             ],
             elasticsearch: {
-              cluster: serviceApiKeyPrivileges.cluster,
-              indices: serviceApiKeyPrivileges.indices,
+              cluster: getServiceApiKeyPrivileges(false).cluster,
+              indices: getServiceApiKeyPrivileges(false).indices,
             },
           });
 
@@ -562,8 +562,8 @@ export default function ({ getService }: FtrProviderContext) {
               },
             ],
             elasticsearch: {
-              cluster: serviceApiKeyPrivileges.cluster,
-              indices: serviceApiKeyPrivileges.indices,
+              cluster: getServiceApiKeyPrivileges(false).cluster,
+              indices: getServiceApiKeyPrivileges(false).indices,
             },
           });