Revert "Revert "Allow Kibana to restrict the usage of JWT for a prede…

…fined set of routes only. (elastic#163806)""

This reverts commit 033fbbd.

.github/CODEOWNERS

@@ -595,7 +595,7 @@ packages/kbn-search-api-panels @elastic/enterprise-search-frontend
 examples/search_examples @elastic/kibana-data-discovery
 packages/kbn-search-response-warnings @elastic/kibana-data-discovery
 x-pack/plugins/searchprofiler @elastic/platform-deployment-management
-x-pack/test/security_api_integration/packages/helpers @elastic/kibana-core
+x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security
 x-pack/plugins/security @elastic/kibana-security
 x-pack/plugins/security_solution_ess @elastic/security-solution
 x-pack/test/cases_api_integration/common/plugins/security_solution @elastic/response-ops

packages/core/status/core-status-server-internal/src/routes/status.ts

@@ -82,7 +82,10 @@ export const registerStatusRoute = ({
       path: '/api/status',
       options: {
         authRequired: 'optional',
-        tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page
+        // The `api` tag ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page.
+        // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to
+        // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly.
+        tags: ['api', 'security:acceptJWT'],
         access: 'public', // needs to be public to allow access from "system" users like k8s readiness probes.
       },
       validate: {

src/plugins/usage_collection/server/routes/stats/stats.ts

@@ -55,7 +55,10 @@ export function registerStatsRoute({
       path: '/api/stats',
       options: {
         authRequired: !config.allowAnonymous,
-        tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page
+        // The `api` tag ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page.
+        // The `security:acceptJWT` tag allows route to be accessed with JWT credentials. It points to
+        // ROUTE_TAG_ACCEPT_JWT from '@kbn/security-plugin/server' that cannot be imported here directly.
+        tags: ['api', 'security:acceptJWT'],
         access: 'public', // needs to be public to allow access from "system" users like metricbeat.
       },
       validate: {

x-pack/plugins/security/server/authentication/authenticator.test.ts

@@ -62,12 +62,14 @@ function getMockOptions({
   selector,
   accessAgreementMessage,
   customLogoutURL,
+  configContext = {},
 }: {
   providers?: Record<string, unknown> | string[];
   http?: Partial<AuthenticatorOptions['config']['authc']['http']>;
   selector?: AuthenticatorOptions['config']['authc']['selector'];
   accessAgreementMessage?: string;
   customLogoutURL?: string;
+  configContext?: Record<string, unknown>;
 } = {}) {
   const auditService = auditServiceMock.create();
   auditLogger = auditLoggerMock.create();
@@ -86,10 +88,10 @@ function getMockOptions({
     loggers: loggingSystemMock.create(),
     getServerBaseURL: jest.fn(),
     config: createConfig(
-      ConfigSchema.validate({
-        authc: { selector, providers, http },
-        ...accessAgreementObj,
-      }),
+      ConfigSchema.validate(
+        { authc: { selector, providers, http }, ...accessAgreementObj },
+        configContext
+      ),
       loggingSystemMock.create().get(),
       { isTLSEnabled: false }
     ),
@@ -317,6 +319,23 @@ describe('Authenticator', () => {
         });
       });
 
+      it('includes JWT options if specified', () => {
+        new Authenticator(
+          getMockOptions({
+            providers: { basic: { basic1: { order: 0 } } },
+            http: { jwt: { taggedRoutesOnly: true } },
+            configContext: { serverless: true },
+          })
+        );
+
+        expect(
+          jest.requireMock('./providers/http').HTTPAuthenticationProvider
+        ).toHaveBeenCalledWith(expect.anything(), {
+          supportedSchemes: new Set(['apikey', 'bearer', 'basic']),
+          jwt: { taggedRoutesOnly: true },
+        });
+      });
+
       it('does not include additional schemes if `autoSchemesEnabled` is disabled', () => {
         new Authenticator(
           getMockOptions({

x-pack/plugins/security/server/authentication/authenticator.ts

@@ -649,7 +649,13 @@ export class Authenticator {
       throw new Error(`Provider name "${options.name}" is reserved.`);
     }
 
-    this.providers.set(options.name, new HTTPAuthenticationProvider(options, { supportedSchemes }));
+    this.providers.set(
+      options.name,
+      new HTTPAuthenticationProvider(options, {
+        supportedSchemes,
+        jwt: this.options.config.authc.http.jwt,
+      })
+    );
   }
 
   /**

x-pack/plugins/security/server/authentication/providers/http.test.ts

@@ -15,6 +15,7 @@ import { mockAuthenticationProviderOptions } from './base.mock';
 import { HTTPAuthenticationProvider } from './http';
 import { mockAuthenticatedUser } from '../../../common/model/authenticated_user.mock';
 import { securityMock } from '../../mocks';
+import { ROUTE_TAG_ACCEPT_JWT } from '../../routes/tags';
 import { AuthenticationResult } from '../authentication_result';
 import { DeauthenticationResult } from '../deauthentication_result';
 
@@ -144,6 +145,113 @@ describe('HTTPAuthenticationProvider', () => {
       }
     });
 
+    it('succeeds for JWT authentication if not restricted to tagged routes.', async () => {
+      const header = 'Bearer header.body.signature';
+      const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } });
+      const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } });
+
+      const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient();
+      mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user);
+      mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient);
+      mockOptions.client.asScoped.mockClear();
+
+      const provider = new HTTPAuthenticationProvider(mockOptions, {
+        supportedSchemes: new Set(['bearer']),
+      });
+
+      await expect(provider.authenticate(request)).resolves.toEqual(
+        AuthenticationResult.succeeded({
+          ...user,
+          authentication_provider: { type: 'http', name: 'http' },
+        })
+      );
+
+      expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } });
+
+      expect(request.headers.authorization).toBe(header);
+    });
+
+    it('succeeds for non-JWT authentication if JWT restricted to tagged routes.', async () => {
+      const header = 'Basic xxx';
+      const user = mockAuthenticatedUser();
+      const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } });
+
+      const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient();
+      mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user);
+      mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient);
+      mockOptions.client.asScoped.mockClear();
+
+      const provider = new HTTPAuthenticationProvider(mockOptions, {
+        supportedSchemes: new Set(['bearer', 'basic']),
+        jwt: { taggedRoutesOnly: true },
+      });
+
+      await expect(provider.authenticate(request)).resolves.toEqual(
+        AuthenticationResult.succeeded({
+          ...user,
+          authentication_provider: { type: 'http', name: 'http' },
+        })
+      );
+
+      expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } });
+
+      expect(request.headers.authorization).toBe(header);
+    });
+
+    it('succeeds for JWT authentication if restricted to tagged routes and route is tagged.', async () => {
+      const header = 'Bearer header.body.signature';
+      const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } });
+      const request = httpServerMock.createKibanaRequest({
+        headers: { authorization: header },
+        routeTags: [ROUTE_TAG_ACCEPT_JWT],
+      });
+
+      const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient();
+      mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user);
+      mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient);
+      mockOptions.client.asScoped.mockClear();
+
+      const provider = new HTTPAuthenticationProvider(mockOptions, {
+        supportedSchemes: new Set(['bearer']),
+        jwt: { taggedRoutesOnly: true },
+      });
+
+      await expect(provider.authenticate(request)).resolves.toEqual(
+        AuthenticationResult.succeeded({
+          ...user,
+          authentication_provider: { type: 'http', name: 'http' },
+        })
+      );
+
+      expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } });
+
+      expect(request.headers.authorization).toBe(header);
+    });
+
+    it('fails for JWT authentication if restricted to tagged routes and route is NOT tagged.', async () => {
+      const header = 'Bearer header.body.signature';
+      const user = mockAuthenticatedUser({ authentication_realm: { name: 'jwt1', type: 'jwt' } });
+      const request = httpServerMock.createKibanaRequest({ headers: { authorization: header } });
+
+      const mockScopedClusterClient = elasticsearchServiceMock.createScopedClusterClient();
+      mockScopedClusterClient.asCurrentUser.security.authenticate.mockResponse(user);
+      mockOptions.client.asScoped.mockReturnValue(mockScopedClusterClient);
+      mockOptions.client.asScoped.mockClear();
+
+      const provider = new HTTPAuthenticationProvider(mockOptions, {
+        supportedSchemes: new Set(['bearer']),
+        jwt: { taggedRoutesOnly: true },
+      });
+
+      await expect(provider.authenticate(request)).resolves.toEqual(
+        AuthenticationResult.notHandled()
+      );
+
+      expectAuthenticateCall(mockOptions.client, { headers: { authorization: header } });
+
+      expect(request.headers.authorization).toBe(header);
+    });
+
     it('fails if authentication via `authorization` header with supported scheme fails.', async () => {
       const failureReason = new errors.ResponseError(securityMock.createApiResponse({ body: {} }));
       for (const { schemes, header } of [

x-pack/plugins/security/server/authentication/providers/http.ts

@@ -9,12 +9,22 @@ import type { KibanaRequest } from '@kbn/core/server';
 
 import type { AuthenticationProviderOptions } from './base';
 import { BaseAuthenticationProvider } from './base';
+import { ROUTE_TAG_ACCEPT_JWT } from '../../routes/tags';
 import { AuthenticationResult } from '../authentication_result';
 import { DeauthenticationResult } from '../deauthentication_result';
 import { HTTPAuthorizationHeader } from '../http_authentication';
 
+/**
+ * A type-string of the Elasticsearch JWT realm.
+ */
+const JWT_REALM_TYPE = 'jwt';
+
 interface HTTPAuthenticationProviderOptions {
   supportedSchemes: Set<string>;
+  jwt?: {
+    // When set, only routes marked with `ROUTE_TAG_ACCEPT_JWT` tag will accept JWT as a means of authentication.
+    taggedRoutesOnly: boolean;
+  };
 }
 
 /**
@@ -32,6 +42,11 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider {
    */
   private readonly supportedSchemes: Set<string>;
 
+  /**
+   * Options relevant to the JWT authentication.
+   */
+  private readonly jwt: HTTPAuthenticationProviderOptions['jwt'];
+
   constructor(
     protected readonly options: Readonly<AuthenticationProviderOptions>,
     httpOptions: Readonly<HTTPAuthenticationProviderOptions>
@@ -44,6 +59,7 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider {
     this.supportedSchemes = new Set(
       [...httpOptions.supportedSchemes].map((scheme) => scheme.toLowerCase())
     );
+    this.jwt = httpOptions.jwt;
   }
 
   /**
@@ -79,6 +95,23 @@ export class HTTPAuthenticationProvider extends BaseAuthenticationProvider {
       this.logger.debug(
         `Request to ${request.url.pathname}${request.url.search} has been authenticated via authorization header with "${authorizationHeader.scheme}" scheme.`
       );
+
+      // If Kibana is configured to restrict JWT authentication only to selected routes, ensure that the route is marked
+      // with the `ROUTE_TAG_ACCEPT_JWT` tag to bypass that restriction.
+      if (
+        user.authentication_realm.type === JWT_REALM_TYPE &&
+        this.jwt?.taggedRoutesOnly &&
+        !request.route.options.tags.includes(ROUTE_TAG_ACCEPT_JWT)
+      ) {
+        // Log a portion of the JWT signature to make debugging easier.
+        const jwtExcerpt = authorizationHeader.credentials.slice(-10);
+        this.logger.error(
+          `Attempted to authenticate with JWT credentials (…${jwtExcerpt}) against ${request.url.pathname}${request.url.search}, but it's not allowed. ` +
+            `Ensure that the route is defined with the "${ROUTE_TAG_ACCEPT_JWT}" tag.`
+        );
+        return AuthenticationResult.notHandled();
+      }
+
       return AuthenticationResult.succeeded(user);
     } catch (err) {
       this.logger.debug(

x-pack/plugins/security/server/config.test.ts

@@ -183,6 +183,68 @@ describe('config schema', () => {
         "showNavLinks": true,
       }
     `);
+
+    expect(ConfigSchema.validate({}, { serverless: true, dist: true })).toMatchInlineSnapshot(`
+      Object {
+        "audit": Object {
+          "enabled": false,
+        },
+        "authc": Object {
+          "http": Object {
+            "autoSchemesEnabled": true,
+            "enabled": true,
+            "jwt": Object {
+              "taggedRoutesOnly": true,
+            },
+            "schemes": Array [
+              "apikey",
+              "bearer",
+            ],
+          },
+          "providers": Object {
+            "anonymous": undefined,
+            "basic": Object {
+              "basic": Object {
+                "accessAgreement": undefined,
+                "description": undefined,
+                "enabled": true,
+                "hint": undefined,
+                "icon": undefined,
+                "order": 0,
+                "session": Object {
+                  "idleTimeout": undefined,
+                  "lifespan": undefined,
+                },
+                "showInSelector": true,
+              },
+            },
+            "kerberos": undefined,
+            "oidc": undefined,
+            "pki": undefined,
+            "saml": undefined,
+            "token": undefined,
+          },
+          "selector": Object {},
+        },
+        "cookieName": "sid",
+        "enabled": true,
+        "loginAssistanceMessage": "",
+        "public": Object {},
+        "secureCookies": false,
+        "session": Object {
+          "cleanupInterval": "PT1H",
+          "idleTimeout": "P3D",
+          "lifespan": "P30D",
+        },
+        "showInsecureClusterWarning": true,
+        "showNavLinks": true,
+        "ui": Object {
+          "roleManagementEnabled": true,
+          "roleMappingManagementEnabled": true,
+          "userManagementEnabled": true,
+        },
+      }
+    `);
   });
 
   it('should throw error if xpack.security.encryptionKey is less than 32 characters', () => {
@@ -1412,6 +1474,34 @@ describe('config schema', () => {
     });
   });
 
+  describe('authc.http', () => {
+    it('should not allow xpack.security.authc.http.jwt.* to be configured outside of the serverless context', () => {
+      expect(() =>
+        ConfigSchema.validate(
+          { authc: { http: { jwt: { taggedRoutesOnly: false } } } },
+          { serverless: false }
+        )
+      ).toThrowErrorMatchingInlineSnapshot(
+        `"[authc.http.jwt]: a value wasn't expected to be present"`
+      );
+    });
+
+    it('should allow xpack.security.authc.http.jwt.* to be configured inside of the serverless context', () => {
+      expect(
+        ConfigSchema.validate(
+          { authc: { http: { jwt: { taggedRoutesOnly: false } } } },
+          { serverless: true }
+        ).ui
+      ).toMatchInlineSnapshot(`
+          Object {
+            "roleManagementEnabled": true,
+            "roleMappingManagementEnabled": true,
+            "userManagementEnabled": true,
+          }
+        `);
+    });
+  });
+
   describe('ui', () => {
     it('should not allow xpack.security.ui.* to be configured outside of the serverless context', () => {
       expect(() =>