WIP: New tuf api metadata for dev initialization

Signed-off-by: Kairo de Araujo <[email protected]>

.gitignore

@@ -20,7 +20,7 @@ docker-compose.override.yaml
 
 node_modules/
 
-dev/tuf.*
+dev/tufkeys/
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz

Makefile

@@ -1,6 +1,5 @@
 DB := example
 IPYTHON := no
-LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
 WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
@@ -80,14 +79,15 @@ initdb:
 	docker-compose run web python -m warehouse sponsors populate-db
 
 inittuf:
-	$(WAREHOUSE_CLI) tuf keypair --name root --path /opt/warehouse/src/dev/tuf.root
-	$(WAREHOUSE_CLI) tuf keypair --name snapshot --path /opt/warehouse/src/dev/tuf.snapshot
-	$(WAREHOUSE_CLI) tuf keypair --name targets --path /opt/warehouse/src/dev/tuf.targets
-	$(WAREHOUSE_CLI) tuf keypair --name timestamp --path /opt/warehouse/src/dev/tuf.timestamp
-	$(WAREHOUSE_CLI) tuf keypair --name bins --path /opt/warehouse/src/dev/tuf.bins
-	$(WAREHOUSE_CLI) tuf keypair --name bin-n --path /opt/warehouse/src/dev/tuf.bin-n
-	$(WAREHOUSE_CLI) tuf new-repo
-	$(WAREHOUSE_CLI) tuf build-targets
+	$(WAREHOUSE_CLI) tuf dev keypair --name root --path /opt/warehouse/src/dev/tufkeys/root
+	$(WAREHOUSE_CLI) tuf dev keypair --name snapshot --path /opt/warehouse/src/dev/tufkeys/snapshot
+	$(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets
+	$(WAREHOUSE_CLI) tuf dev keypair --name timestamp --path /opt/warehouse/src/dev/tufkeys/timestamp
+	$(WAREHOUSE_CLI) tuf dev keypair --name bins --path /opt/warehouse/src/dev/tufkeys/bins
+	$(WAREHOUSE_CLI) tuf dev keypair --name bin-n --path /opt/warehouse/src/dev/tufkeys/bin-n
+	$(WAREHOUSE_CLI) tuf dev new-repo
+	$(WAREHOUSE_CLI) tuf admin delegate-targets-roles
+	$(WAREHOUSE_CLI) tuf dev add-targets
 
 reindex:
 	docker-compose run --rm web python -m warehouse search reindex
@@ -97,6 +97,7 @@ shell:
 
 clean:
 	rm -rf dev/*.sql
+	rm -rf dev/tufkeys
 
 purge: stop clean
 	rm -rf .state

dev/environment

@@ -49,12 +49,12 @@ GITHUB_TOKEN_SCANNING_META_API_URL="http://notgithub:8000/meta/public_keys/token
 TWOFACTORREQUIREMENT_ENABLED=true
 TWOFACTORMANDATE_AVAILABLE=true
 TWOFACTORMANDATE_ENABLED=true
-TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev
-TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService
-TUF_REPO_BACKEND=warehouse.tuf.services.LocalRepositoryService repo.path=/opt/warehouse/src/warehouse/tuf/dist
+
+TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev/tufkeys
+TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService repo.path=/var/opt/warehouse/tuf_metadata/ url=http://localhost:9001/metadata/{path}
 TUF_ROOT_SECRET="an insecure private key password"
 TUF_SNAPSHOT_SECRET="an insecure private key password"
 TUF_TARGETS_SECRET="an insecure private key password"
 TUF_TIMESTAMP_SECRET="an insecure private key password"
 TUF_BINS_SECRET="an insecure private key password"
-TUF_BIN_N_SECRET="an insecure private key password"
+TUF_BIN_N_SECRET="an insecure private key password"

docker-compose.yml

@@ -4,6 +4,7 @@ volumes:
   simple:
   packages:
   sponsorlogos:
+  tuf_metadata:
   vault:
 
 services:
@@ -79,6 +80,7 @@ services:
       # The :z option fixes permission issues with SELinux by setting a
       # permissive security context.
       - ./dev:/opt/warehouse/src/dev:z
+      - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z
       - ./docs:/opt/warehouse/src/docs:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
       - ./tests:/opt/warehouse/src/tests:z
@@ -87,6 +89,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
       - ./bin:/opt/warehouse/src/bin:z
       - ./requirements:/opt/warehouse/src/requirements:z
     ports:
@@ -103,6 +106,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/metadata
     ports:
       - "9001:9001"
 
@@ -115,6 +119,7 @@ services:
     volumes:
       - ./dev:/opt/warehouse/src/dev:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
     env_file: dev/environment
     environment:
       C_FORCE_ROOT: "1"

requirements/main.in

@@ -49,6 +49,7 @@ requests
 requests-aws4auth
 redis>=2.8.0,<5.0.0
 rfc3986
+securesystemslib
 sentry-sdk
 setuptools
 sqlalchemy[asyncio]>=0.9,<1.5.0  # https://github.com/pypa/warehouse/pull/9228
@@ -58,7 +59,7 @@ stdlib-list
 structlog
 transaction
 trove-classifiers
-tuf==0.15.0
+tuf==1.0.0
 typeguard
 webauthn>=1.0.0,<2.0.0
 whitenoise

requirements/main.txt

@@ -1053,6 +1053,7 @@ requests==2.27.1 \
     #   google-cloud-storage
     #   premailer
     #   requests-aws4auth
+    #   tuf
 requests-aws4auth==1.1.1 \
     --hash=sha256:c0883346ce30b5018903a67da88df72f73ff06e1a320845bba9cd85e811ba0ba \
     --hash=sha256:dfd9f930ffde48a756b72b55698a8522875ea6358dcffbcc44a66700ace31783
@@ -1069,6 +1070,12 @@ s3transfer==0.5.1 \
     --hash=sha256:25c140f5c66aa79e1ac60be50dcd45ddc59e83895f062a3aab263b870102911f \
     --hash=sha256:69d264d3e760e569b78aaa0f22c97e955891cd22e32b10c51f784eeda4d9d10a
     # via boto3
+securesystemslib==0.22.0 \
+    --hash=sha256:2f58ca1ee30fde5401300fe3b3841adcf7b4369674247fa63b258e07e1f52fd2 \
+    --hash=sha256:c3fc41ac32fe8bc9744b89e6ce2ebca45f4417ca737beb766a41c6cb21935662
+    # via
+    #   -r requirements/main.in
+    #   tuf
 sentry-sdk==1.5.5 \
     --hash=sha256:3817274fba2498c8ebf6b896ee98ac916c5598706340573268c07bf2bb30d831 \
     --hash=sha256:98fd155fa5d5fec1dbabed32a1a4ae2705f1edaa5dae4e7f7b62a384ba30e759
@@ -1173,6 +1180,10 @@ trove-classifiers==2022.2.16 \
     --hash=sha256:d985eeadb24c8a40d1fa121317466e2ae73ae8ad9abc8cab69371212c40b085f \
     --hash=sha256:dcf20b1de0f56308dd107caaa5175e72a950bbb12c95deedac6a7cd429ecd694
     # via -r requirements/main.in
+tuf==1.0.0 \
+    --hash=sha256:6d967f992d494678b684a6f5f03d948130e2d9956310c7af74fa2f0296e8ec4d \
+    --hash=sha256:ec74150954d56e206fa72f3b0234ea802649370e528dea5a83e507bd408fda82
+    # via -r requirements/main.in
 typeguard==2.13.3 \
     --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \
     --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1

tests/unit/test_config.py

@@ -393,7 +393,6 @@ def __init__(self):
         ),
     ]
     assert configurator_obj.add_static_view.calls == [
-        pretend.call("tuf", "warehouse:tuf/dist/metadata.staged/"),
         pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000),
     ]
     assert configurator_obj.add_cache_buster.calls == [

tests/unit/test_routes.py

@@ -26,6 +26,7 @@ def __init__(self):
                 settings={
                     "docs.url": docs_route_url,
                     "files.url": "https://files.example.com/packages/{path}",
+                    "tuf.url": "https://files.example.com/metadata/{path}",
                 }
             )
             if warehouse:
@@ -337,6 +338,7 @@ def add_policy(name, filename):
             domain=warehouse,
         ),
         pretend.call("packaging.file", "https://files.example.com/packages/{path}"),
+        pretend.call("tuf.metadata", "https://files.example.com/metadata/{path}"),
         pretend.call("ses.hook", "/_/ses-hook/", domain=warehouse),
         pretend.call("rss.updates", "/rss/updates.xml", domain=warehouse),
         pretend.call("rss.packages", "/rss/packages.xml", domain=warehouse),
@@ -425,7 +427,6 @@ def add_policy(name, filename):
             view_kw={"has_translations": True},
         ),
     ]
-
     assert config.add_redirect.calls == [
         pretend.call("/sponsor/", "/sponsors/", domain=warehouse),
         pretend.call("/u/{username}/", "/user/{username}/", domain=warehouse),
@@ -440,6 +441,11 @@ def add_policy(name, filename):
             "https://files.example.com/packages/{path}",
             domain=warehouse,
         ),
+        pretend.call(
+            '/metadata/{path:.*}',
+            'https://files.example.com/metadata/{path}',
+            domain=warehouse,
+        )
     ]
 
     assert config.add_pypi_action_route.calls == [

tests/unit/tuf/test_services.py

@@ -0,0 +1,45 @@
+import pretend
+
+from tuf.api.metadata import Key
+from zope.interface.verify import verifyClass
+
+from warehouse.tuf.interfaces import IKeyService
+from warehouse.tuf.services import LocalKeyService
+
+
+class TestLocalLocalKeyService:
+    def test_verify_service(self):
+        assert verifyClass(IKeyService, LocalKeyService)
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"tuf.key.path": "/tuf/key/path/"})
+        )
+        service = LocalKeyService.create_service(None, request)
+        assert service._key_path == "/tuf/key/path/"
+
+    def test_basic_init(self, db_request):
+        service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request)
+        assert service._key_path == "/opt/warehouse/src/dev/tufkeys"
+
+    def test_get_private_key(self, db_request, monkeypatch):
+        service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request)
+
+        expected_priv_key_dict = {
+            "keytype": "ed25519",
+            "scheme": "ed25519",
+            "keyval": {
+                "public": "720a9a588deefd533c36da9b071f7c7b4d08984e87bfc5a18f34618e438434c7"
+            },
+            "keyid": "2de4eb9afe9fb7307f1dd0869a7aec2235d3418bd63f4214d3ba7d23b516f23e",
+            "keyid_hash_algorithms": ["sha256", "sha512"],
+        }
+        db_request.registry.settings['tuf.root.secret'] = 'tuf.root.secret'
+        monkeypatch.setattr(
+            "warehouse.tuf.services.import_ed25519_privatekey_from_file",
+            lambda *a, **kw: expected_priv_key_dict,
+        )
+
+        root_keyid = service.get("root", "private")
+
+        assert root_keyid == expected_priv_key_dict