Interfaces and services for JWK management

requirements/main.in

@@ -43,6 +43,7 @@ pyramid_rpc>=0.7
 pyramid_services>=2.1
 pyramid_tm>=0.12
 python-slugify
+PyJWT[crypto]>=2.3.0
 readme-renderer[md]>=0.7.0
 requests
 requests-aws4auth

requirements/main.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/main.txt requirements/main.in
+#    pip-compile --allow-unsafe --generate-hashes --output-file=main.txt main.in
 #
 alembic==1.7.5 \
     --hash=sha256:7c328694a2e68f03ee971e63c3bd885846470373a5b532cf2c9f1601c413b153 \
@@ -284,6 +284,7 @@ cryptography==36.0.1 \
     --hash=sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee
     # via
     #   -r main.in
+    #   pyjwt
     #   pyopenssl
     #   webauthn
 cssselect==1.1.0 \
@@ -883,6 +884,10 @@ pygments==2.10.0 \
     --hash=sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380 \
     --hash=sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6
     # via readme-renderer
+pyjwt[crypto]==2.3.0 \
+    --hash=sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41 \
+    --hash=sha256:e0c4bb8d9f0af0c7f5b1ec4c5036309617d03d56932877f2f7a0beeb5318322f
+    # via -r main.in
 pymacaroons==0.13.0 \
     --hash=sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8 \
     --hash=sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907

warehouse/cli/oidc.py

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from warehouse.cli import warehouse
+from warehouse.oidc.tasks import update_oidc_jwks as _update_oidc_jwks
+
+
+@warehouse.group()  # pragma: no branch
+def oidc():
+    """
+    Manage the Warehouse OIDC components.
+    """
+
+
+@oidc.command()
+@click.pass_obj
+def update_oidc_jwks(config):
+    """
+    Update Warehouse's JWK sets for all known OIDC providers.
+    """
+
+    request = config.task(_update_oidc_jwks).get_request()
+    config.task(_update_oidc_jwks).run(request)
+
+
+@oidc.command()
+@click.pass_obj
+@click.option(
+    "--provider", "provider_", help="the name of the provider to list JWKs for"
+)
+def list_jwks(config, provider_):
+    """
+    Dump a JSON blob of all JWK sets known to Warehouse for the given provider
+    """
+
+    from warehouse.oidc.services import JWKService
+
+    jwk_service = JWKService.create_service(None, config)
+
+    print(jwk_service.keyset_for_provider(provider_))

warehouse/config.py

@@ -178,6 +178,7 @@ def configure(settings=None):
     maybe_set(settings, "celery.broker_url", "BROKER_URL")
     maybe_set(settings, "celery.result_url", "REDIS_URL")
     maybe_set(settings, "celery.scheduler_url", "REDIS_URL")
+    maybe_set(settings, "oidc.jwk_cache_url", "REDIS_URL")
     maybe_set(settings, "database.url", "DATABASE_URL")
     maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
     maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_SIX_URL")
@@ -458,6 +459,9 @@ def configure(settings=None):
     # Register support for Macaroon based authentication
     config.include(".macaroons")
 
+    # Register support for OIDC provider based authentication
+    config.include(".oidc")
+
     # Register support for malware checks
     config.include(".malware")
 

warehouse/oidc/__init__.py

@@ -0,0 +1,24 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from celery.schedules import crontab
+
+from warehouse.oidc.interfaces import IJWKService
+from warehouse.oidc.services import JWKService
+
+
+def includeme(config):
+    config.register_service_factory(JWKService.create_service, IJWKService)
+
+    from warehouse.oidc.tasks import update_oidc_jwks
+
+    config.add_periodic_task(crontab(minute=0, hour=8), update_oidc_jwks)

warehouse/oidc/interfaces.py

@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from zope.interface import Interface
+
+
+class IJWKService(Interface):
+    def create_service(context, request):
+        """
+        Create the service, given the context and request for which it is
+        being created.
+        """
+        pass
+
+    def fetch_keysets():
+        """
+        Fetch the JWKs known to Warehouse and yield them as tuples of
+        `(provider-name, key-list)`.
+        """
+        pass
+
+    def keyset_for_provider(provider):
+        """
+        Return a list of JWKs for the given provider.
+        """
+        pass

warehouse/oidc/services.py

@@ -0,0 +1,69 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+import redis
+import requests
+
+from zope.interface import implementer
+
+from warehouse.oidc.interfaces import IJWKService
+from warehouse.utils import oidc
+
+logger = logging.getLogger(__name__)
+
+
+@implementer(IJWKService)
+class JWKService:
+    def __init__(self, config):
+        self._config = config
+
+    @classmethod
+    def create_service(cls, _context, config):
+        return cls(config)
+
+    def fetch_keysets(self):
+        for provider, oidc_url in oidc.OIDC_PROVIDERS.items():
+            resp = requests.get(oidc_url)
+
+            # For whatever reason, an OIDC provider's configuration URL might be
+            # offline. We don't want to completely explode here, since other
+            # providers might still be online (and need updating), so we spit
+            # out an error and continue instead of raising.
+            if not resp.ok:
+                logger.error(
+                    f"error querying OIDC configuration for {provider}: {oidc_url}"
+                )
+                continue
+
+            oidc_conf = resp.json()
+            jwks_url = oidc_conf["jwks_uri"]
+
+            resp = requests.get(jwks_url)
+
+            # Same reasoning as above.
+            if not resp.ok:
+                logger.error(f"error querying JWKS JSON for {provider}: {jwks_url}")
+                continue
+
+            jwks_conf = resp.json()
+            keys = jwks_conf["keys"]
+
+            yield (provider, keys)
+
+    def keyset_for_provider(self, provider):
+        with redis.StrictRedis.from_url(
+            self._config.registry.settings.get("oidc.jwk_cache_url")
+        ) as r:
+            return json.loads(r.get(oidc.jwk_cache_key(provider)))

warehouse/oidc/tasks.py

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import redis
+
+from warehouse import tasks
+from warehouse.oidc.interfaces import IJWKService
+from warehouse.utils import oidc
+
+
+@tasks.task(bind=True, ignore_result=True, acks_late=True)
+def update_oidc_jwks(self, request):
+    with redis.StrictRedis.from_url(
+        request.registry.settings.get("oidc.jwk_cache_url")
+    ) as r:
+        jwk_service = request.find_service(IJWKService)
+        for (provider, keys) in jwk_service.fetch_keysets():
+            r.set(oidc.jwk_cache_key(provider), json.dumps(keys))

warehouse/tasks.py

@@ -207,7 +207,10 @@ def includeme(config):
             Queue("default", routing_key="task.#"),
             Queue("malware", routing_key="malware.#"),
         ),
-        task_routes={"warehouse.malware.tasks.*": {"queue": "malware"}},
+        task_routes={
+            "warehouse.malware.tasks.*": {"queue": "malware"},
+            "warehouse.oidc.tasks.*": {"queue": "oidc"},
+        },
         task_serializer="json",
         worker_disable_rate_limits=True,
         REDBEAT_REDIS_URL=s["celery.scheduler_url"],

warehouse/utils/oidc.py

@@ -0,0 +1,28 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# For now, only RS256 is supported.
+# We probably won't need to support providers with only symmetric keys
+# (e.g. HS256) in the foreseeable future.
+VALID_ALGS = {"RS256"}
+
+OIDC_PROVIDERS = {
+    "github": "https://token.actions.githubusercontent.com/.well-known/openid-configuration",
+}
+
+
+def jwk_cache_key(provider):
+    """
+    Returns a reasonable Redis cache key for the given provider name.
+    """
+    return f"/warehouse/oidc/jwks/{provider}"