Add support for user-linked API keys (#14578)

The primary motivation for converting API keys so that they are explicitly
linked to user accounts is to provide some method for TrueNAS Connect
to use tie in to local or directory services accounts to a credential
that is persistently stored in the TrueNAS Connect keychain.

The legacy API key mechanism was insufficient for this purpose because
the mismatch between user accounts and API keys could lead to
administrators retaining NAS access after account deletion, expiration,
or locking. For more in-depth context and reasoning please refer to the
internal design document NEP-053.

For this purpose, this commit makes the following changes:
1. Legacy API keys are migrated to a user_identifier LEGACY_API_KEY
   which is automatically linked to the root, admin, or truenas_admin
   account depending on server configuration.
2. If the legacy API key granted less than FULL_CONTROL, then the
   API key is migrated with a `revoked` state so that the system
   administrator has an opportunity to review and generate a new
   key and/or service account that provides the correct level of
   access.
3. API key authentication now passes through libpam.
4. user.query results now include an `api_keys` key that contains a
   list of IDs for API keys that exist for the user.
5. Authenticated users with READONLY_ADMIN privilege or greater are
   able to create and manage their own API keys.

During development of this new feature, it was determined that the
original API keys were written with insufficient hashing rounds leading
to the following changes:

6. On successful authentication the stored hash of any LEGACY_API_KEY
   is automatically upgraded to a newer sha512-based standard with
   significantly increased hashing rounds.
7. Dependency on the passlib library was removed.

Since libpam is now used for all non-token authentication methods, a
new auth login endpoint (auth.login_ex) was added to middleware to
facilitate future enhancements for challenge-response authentication
mechanisms. Authentication via username + password + OTP token has been
converted to this new standard. This new endpoint is more closely
aligned with standards and requirements in NIST SP 800-63B. In summary
the following changes were made:

8. An AuthentationContext dataclass was created to house a middleware
   session's PAM context and associated information. An instance of
   this is stored in the session's App object.
9. A new endpoint auth.login_ex was added that is expandable and is
   aware of the server's configured "authentication assurance level".
10. Existing login endpoints (auth.login, auth.login_with_api_key,
   auth.login_with_token) were converted to wrappers around
   auth.login_ex.
11. Initial implementation of session lifetime and inactivity guidelines
   for different authentication assurance levels was added.
12. Challenge-response workflow for username + password + OTP token
   implemented with associated tests.

src/freenas/etc/pam.d/common-account-unix

@@ -0,0 +1,2 @@
+account	requisite			pam_deny.so
+account	required			pam_permit.so

src/middlewared/middlewared/alembic/versions/25.04/2024-10-08_18-48_user_api_key.py

@@ -0,0 +1,42 @@
+"""Convert to user-linked tokens
+
+Revision ID: 8ae49ac78d14
+Revises: 85e5d349cdb1
+Create Date: 2024-10-08 18:48:55.972115+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import json
+
+
+# revision identifiers, used by Alembic.
+revision = '8ae49ac78d14'
+down_revision = '85e5d349cdb1'
+branch_labels = None
+depends_on = None
+
+DEFAULT_ALLOW_LIST = [{"method": "*", "resource": "*"}]
+ENTRY_REVOKED = -1
+
+
+def upgrade():
+    conn = op.get_bind()
+    to_revoke = []
+    for row in conn.execute("SELECT id, allowlist FROM account_api_key").fetchall():
+        try:
+            if json.loads(row['allowlist']) != DEFAULT_ALLOW_LIST:
+                to_revoke.append(str(row['id']))
+        except Exception:
+            to_revoke.append(str(row['id']))
+
+    with op.batch_alter_table('account_api_key', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('user_identifier', sa.String(length=200), nullable=False, server_default='LEGACY_API_KEY'))
+        batch_op.add_column(sa.Column('expiry', sa.Integer(), nullable=False, server_default='0'))
+        batch_op.drop_column('allowlist')
+
+    conn.execute(f"UPDATE account_api_key SET expiry={ENTRY_REVOKED} WHERE id IN ({', '.join(to_revoke)});")
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/alert/source/api_key.py

@@ -0,0 +1,27 @@
+import json
+
+from middlewared.alert.base import Alert, AlertClass, SimpleOneShotAlertClass, AlertCategory, AlertLevel
+
+
+class ApiKeyRevokedAlertClass(AlertClass, SimpleOneShotAlertClass):
+    category = AlertCategory.SYSTEM
+    level = AlertLevel.WARNING
+    title = "API Key Revoked"
+    text = (
+        "%(key_name)s: API key has been revoked and must either be renewed or deleted. "
+        "Once the maintenance is complete, API client configuration must be updated to "
+        "use the renwed API key."
+    )
+
+    async def create(self, args):
+        return Alert(ApiKeyRevokedAlertClass, args, key=args['key_name'])
+
+    async def delete(self, alerts, key_name_set):
+        remaining = []
+        for alert in alerts:
+            if json.loads(alert.key) not in key_name_set:
+                continue
+
+            remaining.append(alert)
+
+        return remaining

src/middlewared/middlewared/api/base/server/app.py

@@ -1,7 +1,7 @@
 import logging
 import uuid
 
-from middlewared.auth import SessionManagerCredentials
+from middlewared.auth import SessionManagerCredentials, AuthenticationContext
 from middlewared.utils.origin import ConnectionOrigin
 
 logger = logging.getLogger(__name__)
@@ -12,6 +12,7 @@ def __init__(self, origin: ConnectionOrigin):
         self.origin = origin
         self.session_id = str(uuid.uuid4())
         self.authenticated = False
+        self.authentication_context: AuthenticationContext = AuthenticationContext()
         self.authenticated_credentials: SessionManagerCredentials | None = None
         self.py_exceptions = False
         self.websocket = False

src/middlewared/middlewared/api/v25_04_0/api_key.py

@@ -4,7 +4,10 @@
 
 from pydantic import Secret, StringConstraints
 
-from middlewared.api.base import BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString
+from middlewared.api.base import (
+    BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString,
+    LocalUsername, RemoteUsername
+)
 
 
 HttpVerb: TypeAlias = Literal["GET", "POST", "PUT", "DELETE", "CALL", "SUBSCRIBE", "*"]
@@ -18,8 +21,13 @@ class AllowListItem(BaseModel):
 class ApiKeyEntry(BaseModel):
     id: int
     name: Annotated[NonEmptyString, StringConstraints(max_length=200)]
+    username: LocalUsername | RemoteUsername
+    user_identifier: int | str
+    keyhash: Secret[str]
     created_at: datetime
-    allowlist: list[AllowListItem]
+    expires_at: datetime | None = None
+    local: bool
+    revoked: bool
 
 
 class ApiKeyEntryWithKey(ApiKeyEntry):
@@ -28,7 +36,11 @@ class ApiKeyEntryWithKey(ApiKeyEntry):
 
 class ApiKeyCreate(ApiKeyEntry):
     id: Excluded = excluded_field()
+    user_identifier: Excluded = excluded_field()
+    keyhash: Excluded = excluded_field()
     created_at: Excluded = excluded_field()
+    local: Excluded = excluded_field()
+    revoked: Excluded = excluded_field()
 
 
 class ApiKeyCreateArgs(BaseModel):
@@ -40,6 +52,7 @@ class ApiKeyCreateResult(BaseModel):
 
 
 class ApiKeyUpdate(ApiKeyCreate, metaclass=ForUpdateMetaclass):
+    username: Excluded = excluded_field()
     reset: bool
 
 
@@ -58,3 +71,11 @@ class ApiKeyDeleteArgs(BaseModel):
 
 class ApiKeyDeleteResult(BaseModel):
     result: Literal[True]
+
+
+class ApiKeyMyKeysArgs(BaseModel):
+    pass
+
+
+class ApiKeyMyKeysResult(BaseModel):
+    result: list[ApiKeyEntry]

src/middlewared/middlewared/api/v25_04_0/auth.py

@@ -1,14 +1,110 @@
 from middlewared.api.base import BaseModel, single_argument_result
+from middlewared.utils.auth import AuthMech, AuthResp
+from pydantic import Field, Secret
+from typing import Literal
 from .user import UserGetUserObjResult
 
 
 class AuthMeArgs(BaseModel):
     pass
 
 
-@single_argument_result
-class AuthMeResult(UserGetUserObjResult.model_fields["result"].annotation):
+class AuthUserInfo(UserGetUserObjResult.model_fields["result"].annotation):
     attributes: dict
     two_factor_config: dict
     privilege: dict
     account_attributes: list[str]
+
+
+class AuthLegacyUsernamePassword(BaseModel):
+    username: str
+    password: Secret[str]
+
+
+class AuthLegacyTwoFactorArgs(AuthLegacyUsernamePassword):
+    pass
+
+
+class AuthLegacyPasswordLoginArgs(AuthLegacyUsernamePassword):
+    otp_token: Secret[str | None] = None
+
+
+class AuthLegacyApiKeyLoginArgs(BaseModel):
+    api_key: Secret[str]
+
+
+class AuthLegacyTokenLoginArgs(BaseModel):
+    token: Secret[str]
+
+
+class AuthLegacyResult(BaseModel):
+    result: bool
+
+
+@single_argument_result
+class AuthMeResult(AuthUserInfo):
+    pass
+
+
+class AuthCommonOptions(BaseModel):
+    user_info: bool = True  # include auth.me in successful result
+
+
+class AuthApiKeyPlain(BaseModel):
+    mechanism: Literal[AuthMech.API_KEY_PLAIN]
+    username: str
+    api_key: Secret[str]
+    login_options: AuthCommonOptions = Field(default=AuthCommonOptions())
+
+
+class AuthPasswordPlain(BaseModel):
+    mechanism: Literal[AuthMech.PASSWORD_PLAIN]
+    username: str
+    password: Secret[str]
+    login_options: AuthCommonOptions = Field(default=AuthCommonOptions())
+
+
+class AuthTokenPlain(BaseModel):
+    mechanism: Literal[AuthMech.TOKEN_PLAIN]
+    token: Secret[str]
+    login_options: AuthCommonOptions = Field(default=AuthCommonOptions())
+
+
+class AuthOTPToken(BaseModel):
+    mechanism: Literal[AuthMech.OTP_TOKEN]
+    otp_token: Secret[str]
+    login_options: AuthCommonOptions = Field(default=AuthCommonOptions())
+
+
+class AuthRespSuccess(BaseModel):
+    response_type: Literal[AuthResp.SUCCESS]
+    user_info: AuthUserInfo | None
+
+
+class AuthRespAuthErr(BaseModel):
+    response_type: Literal[AuthResp.AUTH_ERR]
+
+
+class AuthRespExpired(BaseModel):
+    response_type: Literal[AuthResp.EXPIRED]
+
+
+class AuthRespOTPRequired(BaseModel):
+    response_type: Literal[AuthResp.OTP_REQUIRED]
+    username: str
+
+
+class AuthLoginExArgs(BaseModel):
+    login_data: AuthApiKeyPlain | AuthPasswordPlain | AuthTokenPlain | AuthOTPToken
+
+
+class AuthLoginExResult(BaseModel):
+    result: AuthRespSuccess | AuthRespAuthErr | AuthRespExpired | AuthRespOTPRequired
+
+
+class AuthMechChoicesArgs(BaseModel):
+    pass
+
+
+class AuthMechChoicesResult(BaseModel):
+    result: list[str]

src/middlewared/middlewared/api/v25_04_0/user.py

@@ -57,6 +57,7 @@ class UserEntry(BaseModel):
     twofactor_auth_configured: bool
     sid: str | None
     roles: list[str]
+    api_keys: list[int]
 
 
 class UserCreate(UserEntry):
@@ -70,6 +71,7 @@ class UserCreate(UserEntry):
     twofactor_auth_configured: Excluded = excluded_field()
     sid: Excluded = excluded_field()
     roles: Excluded = excluded_field()
+    api_keys: Excluded = excluded_field()
 
     uid: LocalUID | None = None
     "UNIX UID. If not provided, it is automatically filled with the next one available."