diff --git a/.gitignore b/.gitignore index 8f788ddc636b..f4e885597c79 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ docker-compose.override.yaml node_modules/ -dev/tuf.* +dev/tufkeys/ dev/example.sql dev/prod.sql dev/prod.sql.xz diff --git a/Makefile b/Makefile index 47c7f10943e3..f96efa4b70d1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ DB := example IPYTHON := no -LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))") WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython @@ -80,14 +79,15 @@ initdb: docker-compose run web python -m warehouse sponsors populate-db inittuf: - $(WAREHOUSE_CLI) tuf keypair --name root --path /opt/warehouse/src/dev/tuf.root - $(WAREHOUSE_CLI) tuf keypair --name snapshot --path /opt/warehouse/src/dev/tuf.snapshot - $(WAREHOUSE_CLI) tuf keypair --name targets --path /opt/warehouse/src/dev/tuf.targets - $(WAREHOUSE_CLI) tuf keypair --name timestamp --path /opt/warehouse/src/dev/tuf.timestamp - $(WAREHOUSE_CLI) tuf keypair --name bins --path /opt/warehouse/src/dev/tuf.bins - $(WAREHOUSE_CLI) tuf keypair --name bin-n --path /opt/warehouse/src/dev/tuf.bin-n - $(WAREHOUSE_CLI) tuf new-repo - $(WAREHOUSE_CLI) tuf build-targets + $(WAREHOUSE_CLI) tuf dev keypair --name root --path /opt/warehouse/src/dev/tufkeys/root + $(WAREHOUSE_CLI) tuf dev keypair --name snapshot --path /opt/warehouse/src/dev/tufkeys/snapshot + $(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets + $(WAREHOUSE_CLI) tuf dev keypair --name timestamp --path /opt/warehouse/src/dev/tufkeys/timestamp + $(WAREHOUSE_CLI) tuf dev keypair --name bins --path /opt/warehouse/src/dev/tufkeys/bins + $(WAREHOUSE_CLI) tuf dev keypair --name bin-n --path /opt/warehouse/src/dev/tufkeys/bin-n + $(WAREHOUSE_CLI) tuf dev new-repo + $(WAREHOUSE_CLI) tuf admin delegate-targets-roles + $(WAREHOUSE_CLI) tuf dev add-targets reindex: docker-compose run --rm web python -m warehouse search reindex @@ -97,6 +97,7 @@ shell: clean: rm -rf dev/*.sql + rm -rf dev/tufkeys purge: stop clean rm -rf .state diff --git a/dev/environment b/dev/environment index 4a46a7c1ea81..5d4d3959d530 100644 --- a/dev/environment +++ b/dev/environment @@ -49,12 +49,12 @@ GITHUB_TOKEN_SCANNING_META_API_URL="http://notgithub:8000/meta/public_keys/token TWOFACTORREQUIREMENT_ENABLED=true TWOFACTORMANDATE_AVAILABLE=true TWOFACTORMANDATE_ENABLED=true -TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev -TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService -TUF_REPO_BACKEND=warehouse.tuf.services.LocalRepositoryService repo.path=/opt/warehouse/src/warehouse/tuf/dist + +TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev/tufkeys +TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService repo.path=/var/opt/warehouse/tuf_metadata/ url=http://localhost:9001/metadata/{path} TUF_ROOT_SECRET="an insecure private key password" TUF_SNAPSHOT_SECRET="an insecure private key password" TUF_TARGETS_SECRET="an insecure private key password" TUF_TIMESTAMP_SECRET="an insecure private key password" TUF_BINS_SECRET="an insecure private key password" -TUF_BIN_N_SECRET="an insecure private key password" +TUF_BIN_N_SECRET="an insecure private key password" \ No newline at end of file diff --git a/dev/tufkeys/bin-n b/dev/tufkeys/bin-n deleted file mode 100644 index f1ba72f72ea2..000000000000 --- a/dev/tufkeys/bin-n +++ /dev/null @@ -1 +0,0 @@ -2f0570511d84b133d6e1f875920587a9@@@@100000@@@@dfacbe8d4966935d16b5c9b7910e7b01189f83e0db2d5169eb4d7a2edab91924@@@@f3f67a27b03a24c8b767f5721bc345b5@@@@e9855ff4f41d1ec10877b3476eabd99c86f2162ef16172aaa17200af3ec97a0e7b05a389e3084a8c79aa2756fd999eaa67f5462e37a024b66feba784a3e70850d8d2466165ba86d745cac2a526d44439892f46a3342d1c890589c833f652942283b59441b214625b275de95d16ec199cb4fbbd7fa8b1442153fbac65db18d9c9e1345b37a7fa850d2ffe0d13035f28d68a3b47ddb310750002e8e96e751b633f06e4d9c70fa117de12d848cab845f830e082c51e0ad42342f2c24869b091e3c7dae70410578fbb6877a975983aa3dde6aa699e4e45ba7114c4e373109d2128c5211b3471390cb0ef09da3cd5e552813a906f604038218f7891213cefa10fbb7ba6c01e98c821ba45b69504d5ac2d41feeafccc974d82b89c33d200b6ceb04a7af47f08f8258cfea9a6855d3cbe91bc0892ddaabd0f508a6ed85f811dcff66e0fed066f3607f5a5f1ccecc80ad1ede362 \ No newline at end of file diff --git a/dev/tufkeys/bin-n.pub b/dev/tufkeys/bin-n.pub deleted file mode 100644 index 23af98de62af..000000000000 --- a/dev/tufkeys/bin-n.pub +++ /dev/null @@ -1 +0,0 @@ -{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "ca2694d781367f94974c5176cae6c7290fe3b65f03a5c6331fe500b8c700f3aa"}} \ No newline at end of file diff --git a/dev/tufkeys/bins b/dev/tufkeys/bins deleted file mode 100644 index d72696818560..000000000000 --- a/dev/tufkeys/bins +++ /dev/null @@ -1 +0,0 @@ -c4509bc89725ce7b1467b4d596536564@@@@100000@@@@47c863bddaad67e2482760e0da329f3f9170dee5de9e9265cd23873947b4a373@@@@8333670a562d5b2fa37559bb4372de32@@@@d567e40c7d544b4d41f023d208cacdb76009991fb1dc4681e8b7ebae42d5756c21434937061297dfe264150d3ab9c00985d4217ca18c70904d8c1becc9e172a1677221d09d7f9de5b6778340bf26d305dcf34556066e744abcf96bd489e612d6ea9c483838f21bf06646d250b5d7f380cccf57eb990387baf28c3fd00815b1db3a34418d879718607fb54ccfb1fe20edc06bdb765dce6d3617dba5cc5aff41f5eab62a6e0f8e095a97dda3784dcb95f97af5a1c4de7fb0424d659469305cafb092a209ab78932444ffb497286cb44e4ba360e0025f71fc108e8f3b2fac794a05242ef5167bdcd5e4a1a3573805cb91b8359bcb317eb026c2ee102db096b50b5f6fc478af5146dbcd9e2ada046d6a392854e779ffa4b2bdbb0ea90d41eb6030fd0d98f0aa34510a2a6e9292d5095140864658d8db63701b661ed8ee3b6a428dfb9163e7918942ffb0506f290a76fc4703 \ No newline at end of file diff --git a/dev/tufkeys/bins.pub b/dev/tufkeys/bins.pub deleted file mode 100644 index d0ef047f9a51..000000000000 --- a/dev/tufkeys/bins.pub +++ /dev/null @@ -1 +0,0 @@ -{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "a4361da485b8ae5cb10915d5254d96f05ed3d8d4c87979c5a203504b4b65ecf5"}} \ No newline at end of file diff --git a/dev/tufkeys/root b/dev/tufkeys/root deleted file mode 100644 index ddb1d5081269..000000000000 --- a/dev/tufkeys/root +++ /dev/null @@ -1 +0,0 @@ -83589fb72074fcb8533c8c4414015c30@@@@100000@@@@be3d99c519272cc8b8c1150ca03450c0aac21d084a0dc79966faca4d05787ff1@@@@dccf47ed06f92da01e9de35113477e88@@@@72e37b2acdb2115888975e66990009e6f6a4cdd94ff77473e056d16ef33e9ef6ea450308f1b4962f544a0796d7b7f788b35a6ae33d6cc83e56cc416440716418102d39573692e87350935088bc08b4b82d910731a0f536bebded8f80e49c3df052da2bf657ed6a1f64463da477905bc8a1bc8fb65215653a6efa3e0a74b2fabcb65d96fc1f1c31196878e9eb39eca1cfb709b69a400f159f5e8f8dc9b765590fb5cac705170149b71c0f54951d239eafaa9ea9b8c3ac0897e57e5c8f027f8d160d91e3995da00535ced37ff41bccd31e7ed5076c22b448f9a7e701a2024de56b839fd7be4689c23f8b81102fe4711585300862b01f499408295723f995775de33ce69e666883bb25302adb1ce6b21ac710ceeacefacc316aa80432a7d8e9819d78913707b8daef35d03667ab11727e6ea6bc44b9356449f6d6aed53825f2fd1397c68d2669b2b1895874678234a42afd \ No newline at end of file diff --git a/dev/tufkeys/root.pub b/dev/tufkeys/root.pub deleted file mode 100644 index 61e15f87d712..000000000000 --- a/dev/tufkeys/root.pub +++ /dev/null @@ -1 +0,0 @@ -{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "5cb1a1622f72ae901bc2ed25546503bae70b53308833385a11b6630fb52c9bc9"}} \ No newline at end of file diff --git a/dev/tufkeys/snapshot b/dev/tufkeys/snapshot deleted file mode 100644 index 4b899fc99d94..000000000000 --- a/dev/tufkeys/snapshot +++ /dev/null @@ -1 +0,0 @@ -97074b6eccb9d5920192f231249c33dc@@@@100000@@@@3965a3500a3de4f6a052f9f26efd2714affae878daec6df53c50fe20fb3b5732@@@@dbc2d164f88ea8d77b093db20475db0c@@@@d6db7159767d46f6f255a2184b2615df892f067ca0861dba18ccc5b6b7dce95d5d2b377d0d8424d39fb0b337e9bd086a6405804a55e7c8d591f090dc55d38c0820a6bc2ddd27078e1bc482fa4f19a66941cf023680d032da70a8ac6bb5c9fc7454a273d014ef569f7a79f89145c29ade6a5f56752a9e4b5f3a4ee0426680b3d1921a962a6edee5c77661439f7bc95674fbc009ecf903c20fbac786e94164bc43df1fb8536bc6666ce098cfad411c852154e4dd7dda0b9046cd4310d7ae9b822be3cb7b62e66e73718424743e2297b091d2cd8cfce108f1f6a2e17ad382655c517d8131da3a72ac1f2ae8cd760705bd17ec5ff6834c86e8f08fa804152774146a3bda063ae29a01be00723b96e3370d9f046a1287ad4ccaa007cc016d59bf935268cb8b7946bb5b5480962137ed6cb449df43aa83d3d086e4b10e7c33110847c9bc1ee2c24ebbd66e989c6e28c52ac28d \ No newline at end of file diff --git a/dev/tufkeys/snapshot.pub b/dev/tufkeys/snapshot.pub deleted file mode 100644 index 92093748d91e..000000000000 --- a/dev/tufkeys/snapshot.pub +++ /dev/null @@ -1 +0,0 @@ -{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "c19d2c8532667d7784f5b33f5bdccd529b52c5d55754b4ef989a261bf1695431"}} \ No newline at end of file diff --git a/dev/tufkeys/targets b/dev/tufkeys/targets deleted file mode 100644 index a36285e32f8e..000000000000 --- a/dev/tufkeys/targets +++ /dev/null @@ -1 +0,0 @@ -e25efe7833ec8682ff0d09366d264fe5@@@@100000@@@@ecb082633529fc0d392b3d0c8f36972423c91060bbfc1903a8e390deeb650817@@@@3f116f9b022d8e1e3959e99ee800f5cd@@@@45defac0e6ef65003645fd34e3a39a47579e6da578529fe425bd9798e6007cb901d4003e866068a853e17faeba365367f63ead34fac88bca0938f308e2556913c8f9ed56892262704b0a7dcb25bedf61fcd8c72af6eaa4b8630936b704f2aae364313211a32d3bac84d88273fd2e2ee48f1c3d7bef8de74a800124f469bacad916ed3d1242b2c691548b68d3e890a8ef19d3e56817a94e1e508136a8f6cd4d5af4012a355e874bfe446504b75991187f76f521dc3b589d1eae126d978e1e7b268688e1992729cb820f5057352e6c916af4ce448bf0a26d1cf52968a5398c6c516cae831f7f75a7885489fae79fb3a9f8faae0dc32ceaf17106df7834ffc4e28589df0d886a08f51751d2c15999d538197a1e392647ffc81a314e87f936e2087762f0d6004a862789e27021e11e489818320da3bd8801f208deb0b4aa4d1f6319712729bb716c71837ac229d216180ef4 \ No newline at end of file diff --git a/dev/tufkeys/targets.pub b/dev/tufkeys/targets.pub deleted file mode 100644 index fe0f800e39af..000000000000 --- a/dev/tufkeys/targets.pub +++ /dev/null @@ -1 +0,0 @@ -{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "715ea559e2fa80a2bad3c50836c55ce33d256fa7ad5468931cc76a8a10e86d37"}} \ No newline at end of file diff --git a/dev/tufkeys/timestamp b/dev/tufkeys/timestamp deleted file mode 100644 index a77ed1a42473..000000000000 --- a/dev/tufkeys/timestamp +++ /dev/null @@ -1 +0,0 @@ -7a59984187faa4893ceba030f5c5b4a7@@@@100000@@@@7ec97a52365194434c8294e50ceb454222194670104eeacce30846222cdd2756@@@@20d9afcaebb214eb40b3827d6730f318@@@@7763c7a7fd2d9897ad9951149bbd00ced57d9a68339ce0b1a1dbfc7cbb1defdd6d2a87178c140a9de2180e58616f6004a53f384356743b3aad3d54d073e70d8a6c9420ee9e6880405d969f59ddac072235740259868af52a9c0d4582ce3cd0435e9dc0138f0571b02629721f11308e5a6fb26f75d2bd9d298da3bfaad4ec6aaa80d0d81a2e22690d8416b188e132b87bc5ef89d1460e4f7b2c12dac555ad99212b30c73d5f3ef30a77e0eb2f0f82e88c869ef585a8707267be76619c7a3e430563be1d7d1073ed1706bea7d5232e784674c104920d4f44382beeb8d07cfe0d9f5ec0812497d48d2f8b71634a2ae29ca9bf2732d4e0c3216a62b318d11dfc1608f0c9f9e8eff142b45cce4ac7bc91c4967fcd9b3b5484bf016375100391321c54e6a3da85cfb99c6e429da2a586ea15d2f09e2f7acb3e5358bef10f0116b9f3da2aa2c49f9fb07ffcaea4677144bc1969 \ No newline at end of file diff --git a/dev/tufkeys/timestamp.pub b/dev/tufkeys/timestamp.pub deleted file mode 100644 index cece0173a650..000000000000 --- a/dev/tufkeys/timestamp.pub +++ /dev/null @@ -1 +0,0 @@ -{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "5e12fa7b474cb886cdc0cd14c5017b09b3759125ab7e6985bd9c5953063de333"}} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 869bfb5e3cef..5fe5fc2e3ffa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ volumes: simple: packages: sponsorlogos: + tuf_metadata: vault: services: @@ -79,6 +80,7 @@ services: # The :z option fixes permission issues with SELinux by setting a # permissive security context. - ./dev:/opt/warehouse/src/dev:z + - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z - ./docs:/opt/warehouse/src/docs:z - ./warehouse:/opt/warehouse/src/warehouse:z - ./tests:/opt/warehouse/src/tests:z @@ -87,6 +89,7 @@ services: - packages:/var/opt/warehouse/packages - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - tuf_metadata:/var/opt/warehouse/tuf_metadata - ./bin:/opt/warehouse/src/bin:z - ./requirements:/opt/warehouse/src/requirements:z ports: @@ -103,6 +106,7 @@ services: - packages:/var/opt/warehouse/packages - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - tuf_metadata:/var/opt/warehouse/metadata ports: - "9001:9001" @@ -115,6 +119,7 @@ services: volumes: - ./dev:/opt/warehouse/src/dev:z - ./warehouse:/opt/warehouse/src/warehouse:z + - tuf_metadata:/var/opt/warehouse/tuf_metadata env_file: dev/environment environment: C_FORCE_ROOT: "1" diff --git a/requirements/main.in b/requirements/main.in index 32c8c1e08149..87ea11ca4d20 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -49,6 +49,7 @@ requests requests-aws4auth redis>=2.8.0,<5.0.0 rfc3986 +securesystemslib sentry-sdk setuptools sqlalchemy[asyncio]>=0.9,<1.5.0 # https://github.com/pypa/warehouse/pull/9228 @@ -58,7 +59,7 @@ stdlib-list structlog transaction trove-classifiers -tuf==0.15.0 +tuf==1.0.0 typeguard webauthn>=1.0.0,<2.0.0 whitenoise diff --git a/requirements/main.txt b/requirements/main.txt index 64ec7df99c12..4cda6885b029 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1053,6 +1053,7 @@ requests==2.27.1 \ # google-cloud-storage # premailer # requests-aws4auth + # tuf requests-aws4auth==1.1.1 \ --hash=sha256:c0883346ce30b5018903a67da88df72f73ff06e1a320845bba9cd85e811ba0ba \ --hash=sha256:dfd9f930ffde48a756b72b55698a8522875ea6358dcffbcc44a66700ace31783 @@ -1069,6 +1070,12 @@ s3transfer==0.5.1 \ --hash=sha256:25c140f5c66aa79e1ac60be50dcd45ddc59e83895f062a3aab263b870102911f \ --hash=sha256:69d264d3e760e569b78aaa0f22c97e955891cd22e32b10c51f784eeda4d9d10a # via boto3 +securesystemslib==0.22.0 \ + --hash=sha256:2f58ca1ee30fde5401300fe3b3841adcf7b4369674247fa63b258e07e1f52fd2 \ + --hash=sha256:c3fc41ac32fe8bc9744b89e6ce2ebca45f4417ca737beb766a41c6cb21935662 + # via + # -r requirements/main.in + # tuf sentry-sdk==1.5.5 \ --hash=sha256:3817274fba2498c8ebf6b896ee98ac916c5598706340573268c07bf2bb30d831 \ --hash=sha256:98fd155fa5d5fec1dbabed32a1a4ae2705f1edaa5dae4e7f7b62a384ba30e759 @@ -1173,6 +1180,10 @@ trove-classifiers==2022.2.16 \ --hash=sha256:d985eeadb24c8a40d1fa121317466e2ae73ae8ad9abc8cab69371212c40b085f \ --hash=sha256:dcf20b1de0f56308dd107caaa5175e72a950bbb12c95deedac6a7cd429ecd694 # via -r requirements/main.in +tuf==1.0.0 \ + --hash=sha256:6d967f992d494678b684a6f5f03d948130e2d9956310c7af74fa2f0296e8ec4d \ + --hash=sha256:ec74150954d56e206fa72f3b0234ea802649370e528dea5a83e507bd408fda82 + # via -r requirements/main.in typeguard==2.13.3 \ --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \ --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1 diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 48c89950d9de..d36361f4a72b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -393,7 +393,6 @@ def __init__(self): ), ] assert configurator_obj.add_static_view.calls == [ - pretend.call("tuf", "warehouse:tuf/dist/metadata.staged/"), pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000), ] assert configurator_obj.add_cache_buster.calls == [ diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index 634d0a927eb9..503a23e4bfba 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -26,6 +26,7 @@ def __init__(self): settings={ "docs.url": docs_route_url, "files.url": "https://files.example.com/packages/{path}", + "tuf.url": "https://files.example.com/metadata/{path}", } ) if warehouse: @@ -337,6 +338,7 @@ def add_policy(name, filename): domain=warehouse, ), pretend.call("packaging.file", "https://files.example.com/packages/{path}"), + pretend.call("tuf.metadata", "https://files.example.com/metadata/{path}"), pretend.call("ses.hook", "/_/ses-hook/", domain=warehouse), pretend.call("rss.updates", "/rss/updates.xml", domain=warehouse), pretend.call("rss.packages", "/rss/packages.xml", domain=warehouse), @@ -425,7 +427,6 @@ def add_policy(name, filename): view_kw={"has_translations": True}, ), ] - assert config.add_redirect.calls == [ pretend.call("/sponsor/", "/sponsors/", domain=warehouse), pretend.call("/u/{username}/", "/user/{username}/", domain=warehouse), @@ -440,6 +441,11 @@ def add_policy(name, filename): "https://files.example.com/packages/{path}", domain=warehouse, ), + pretend.call( + '/metadata/{path:.*}', + 'https://files.example.com/metadata/{path}', + domain=warehouse, + ) ] assert config.add_pypi_action_route.calls == [ diff --git a/tests/unit/tuf/__init__.py b/tests/unit/tuf/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py new file mode 100644 index 000000000000..683232a194f3 --- /dev/null +++ b/tests/unit/tuf/test_services.py @@ -0,0 +1,45 @@ +import pretend + +from tuf.api.metadata import Key +from zope.interface.verify import verifyClass + +from warehouse.tuf.interfaces import IKeyService +from warehouse.tuf.services import LocalKeyService + + +class TestLocalLocalKeyService: + def test_verify_service(self): + assert verifyClass(IKeyService, LocalKeyService) + + def test_create_service(self): + request = pretend.stub( + registry=pretend.stub(settings={"tuf.key.path": "/tuf/key/path/"}) + ) + service = LocalKeyService.create_service(None, request) + assert service._key_path == "/tuf/key/path/" + + def test_basic_init(self, db_request): + service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request) + assert service._key_path == "/opt/warehouse/src/dev/tufkeys" + + def test_get_private_key(self, db_request, monkeypatch): + service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request) + + expected_priv_key_dict = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "720a9a588deefd533c36da9b071f7c7b4d08984e87bfc5a18f34618e438434c7" + }, + "keyid": "2de4eb9afe9fb7307f1dd0869a7aec2235d3418bd63f4214d3ba7d23b516f23e", + "keyid_hash_algorithms": ["sha256", "sha512"], + } + db_request.registry.settings['tuf.root.secret'] = 'tuf.root.secret' + monkeypatch.setattr( + "warehouse.tuf.services.import_ed25519_privatekey_from_file", + lambda *a, **kw: expected_priv_key_dict, + ) + + root_keyid = service.get("root", "private") + + assert root_keyid == expected_priv_key_dict diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py index 9350759e62e8..fa58fd36f992 100644 --- a/warehouse/cli/tuf.py +++ b/warehouse/cli/tuf.py @@ -10,16 +10,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime - import click -from tuf import repository_tool - from warehouse.cli import warehouse from warehouse.config import Environment from warehouse.tuf import utils -from warehouse.tuf.constants import BIN_N_COUNT, TOPLEVEL_ROLES, Role +from warehouse.tuf.constants import BIN_N_COUNT, Role +from warehouse.tuf.hash_bins import HashBins +from warehouse.tuf.repository import ( + TOP_LEVEL_ROLE_NAMES, + MetadataRepository, + RolesPayload, + TargetsPayload, +) +from warehouse.tuf.utils import ( + repository_bump_bins_ns, + repository_bump_snapshot, + set_expiration_for_role, +) + + +def _make_hash_bins(config): + if config.registry.settings["warehouse.env"] == Environment.development: + number_of_bins = 32 + else: + number_of_bins = BIN_N_COUNT + + return HashBins(number_of_bins) def _make_backsigned_fileinfo_from_file(file): @@ -31,27 +48,13 @@ def _key_service(config): return key_service_class.create_service(None, config) -def _repository_service(config): +def _storage_service(config): repo_service_class = config.maybe_dotted( - config.registry.settings["tuf.repo_backend"] + config.registry.settings["tuf.storage_backend"] ) return repo_service_class.create_service(None, config) -def _set_expiration_for_role(config, role_obj, role_name): - # If we're initializing TUF for development purposes, give - # every role a long expiration time so that developers don't have to - # continually re-initialize it. - if config.registry.settings["warehouse.env"] == Environment.development: - role_obj.expiration = datetime.datetime.now() + datetime.timedelta( - seconds=config.registry.settings["tuf.development_metadata_expiry"] - ) - else: - role_obj.expiration = datetime.datetime.now() + datetime.timedelta( - seconds=config.registry.settings[f"tuf.{role_name}.expiry"] - ) - - @warehouse.group() # pragma: no-branch def tuf(): """ @@ -59,7 +62,14 @@ def tuf(): """ -@tuf.command() +@tuf.group() +def dev(): + """ + TUF Development purposes commands + """ + + +@dev.command() @click.pass_obj @click.option("--name", "name_", help="The name of the TUF role for this keypair") @click.option("--path", "path_", help="The basename of the Ed25519 keypair to generate") @@ -67,110 +77,164 @@ def keypair(config, name_, path_): """ Generate a new TUF keypair, for development purposes. """ - - repository_tool.generate_and_write_ed25519_keypair( - path_, password=config.registry.settings[f"tuf.{name_}.secret"] - ) + utils.create_dev_keys(config.registry.settings[f"tuf.{name_}.secret"], path_) -@tuf.command() +@dev.command() @click.pass_obj def new_repo(config): """ - Initialize the TUF repository from scratch, including a brand new root. + Initialize new TUF repository from scratch, for development purposes. """ - repository = repository_tool.create_new_repository( - config.registry.settings["tuf.repo.path"] - ) + key_service = _key_service(config) + storage_service = _storage_service(config) + metadata_repository = MetadataRepository(storage_service, key_service) + + if metadata_repository._is_initialized: + raise click.ClickException("TUF Metadata Repository already initialized.") + + top_roles_payload = dict() + for role in TOP_LEVEL_ROLE_NAMES: + top_roles_payload[role] = RolesPayload( + expiration=set_expiration_for_role(config, role), + threshold=config.registry.settings[f"tuf.{role}.threshold"], + keys=[key_service.get(role, "private")], + ) + + try: + metadata_repository.initialize(top_roles_payload, True) + except (ValueError, FileExistsError) as err: + raise click.ClickException(str(err)) + + if metadata_repository.is_initialized is False: + raise click.ClickException("TUF Metadata Repository failed to initialized.") + + +@dev.command() +@click.pass_obj +def add_targets(config): + """ + Collect the "paths" for every PyPI package. These are packages already in + existence, so we'll add some additional data to their targets to + indicate that we're back-signing them. + """ + from warehouse.db import Session + from warehouse.packaging.models import File key_service = _key_service(config) - for role in TOPLEVEL_ROLES: - role_obj = getattr(repository, role) - role_obj.threshold = config.registry.settings[f"tuf.{role}.threshold"] - _set_expiration_for_role(config, role_obj, role) - - pubkeys = key_service.pubkeys_for_role(role) - privkeys = key_service.privkeys_for_role(role) - if len(pubkeys) < role_obj.threshold or len(privkeys) < role_obj.threshold: - raise click.ClickException( - f"Unable to initialize TUF repo ({role} needs {role_obj.threshold} keys" - ) + storage_service = _storage_service(config) + metadata_repository = MetadataRepository(storage_service, key_service) + hash_bins = _make_hash_bins(config) + + db = Session(bind=config.registry["sqlalchemy.engine"]) - for pubkey in pubkeys: - role_obj.add_verification_key(pubkey) + payload = dict() + for file in db.query(File).all(): + fileinfo = _make_backsigned_fileinfo_from_file(file) + delegated_role_bin_name = hash_bins.get_delegate(file.path) + target_file = TargetsPayload(fileinfo, file.path) + if payload.get(delegated_role_bin_name) is None: + payload[delegated_role_bin_name] = list() - for privkey in privkeys: - role_obj.load_signing_key(privkey) + payload[delegated_role_bin_name].append(target_file) - repository.mark_dirty(TOPLEVEL_ROLES) - repository.writeall( - consistent_snapshot=True, + metadata_repository.add_targets( + payload, + set_expiration_for_role(config, Role.TIMESTAMP.value), + set_expiration_for_role(config, Role.SNAPSHOT.value), ) -@tuf.command() -@click.pass_obj -def build_targets(config): +@tuf.group() +def admin(): """ - Given an initialized (but empty) TUF repository, create the delegated - targets role (bins) and its hashed bin delegations (each bin-n). + TUF Administrative commands """ - repo_service = _repository_service(config) - repository = repo_service.load_repository() - # Load signing keys. We do this upfront for the top-level roles. +@admin.command() +@click.pass_obj +def bump_snapshot(config): + """ + Bump Snapshot metadata + """ + storage_service = _storage_service(config) key_service = _key_service(config) - for role in ["snapshot", "targets", "timestamp"]: - role_obj = getattr(repository, role) - [role_obj.load_signing_key(k) for k in key_service.privkeys_for_role(role)] + repository_bump_snapshot(config, storage_service, key_service) + click.echo("Snapshot bump finished.") - # NOTE: TUF normally does delegations by path patterns (i.e., globs), but PyPI - # doesn't store its uploads on the same logical host as the TUF repository. - # The last parameter to `delegate` is a special sentinel for this. - repository.targets.delegate( - Role.BINS.value, key_service.pubkeys_for_role(Role.BINS.value), ["*"] - ) - bins_role = repository.targets(Role.BINS.value) - _set_expiration_for_role(config, bins_role, Role.BINS.value) - for privkey in key_service.privkeys_for_role(Role.BINS.value): - bins_role.load_signing_key(privkey) +@admin.command() +@click.pass_obj +def bump_bin_ns(config): + """ + Bump BIN-S roles + """ + storage_service = _storage_service(config) + key_service = _key_service(config) + + repository_bump_bins_ns(config, storage_service, key_service) + click.echo("Hash-bins BIN-S Roles bump finished.") - bins_role.delegate_hashed_bins( - [], - key_service.pubkeys_for_role(Role.BIN_N.value), - BIN_N_COUNT, - ) - dirty_roles = ["snapshot", "targets", "timestamp", Role.BINS.value] - for bin_n_role in bins_role.delegations: - _set_expiration_for_role(config, bin_n_role, Role.BIN_N.value) - dirty_roles.append(bin_n_role.rolename) +@admin.command() +@click.pass_obj +def delegate_targets_roles(config): + """ + Create delegated targets roles (BINS and BIN-N). - for privkey in key_service.privkeys_for_role(Role.BIN_N.value): - for bin_n_role in bins_role.delegations: - bin_n_role.load_signing_key(privkey) + Given an initialized (but empty) TUF repository, create the delegated + targets role (bins) and its hashed bin delegations (each bin-n). + """ - # Collect the "paths" for every PyPI package. These are packages already in - # existence, so we'll add some additional data to their targets to - # indicate that we're back-signing them. - from warehouse.db import Session - from warehouse.packaging.models import File + key_service = _key_service(config) + storage_service = _storage_service(config) + metadata_repository = MetadataRepository(storage_service, key_service) + hash_bins = _make_hash_bins(config) + + # Delegate first targets -> BINS + delegate_roles_payload = dict() + delegate_roles_payload["targets"] = list() + delegate_roles_payload["targets"].append( + RolesPayload( + expiration=set_expiration_for_role(config, Role.BINS.value), + threshold=config.registry.settings[f"tuf.{Role.BINS.value}.threshold"], + keys=[key_service.get(Role.BINS.value, "private")], + delegation_role=Role.BINS.value, + paths=["*/*/*/*"], + ) + ) - db = Session(bind=config.registry["sqlalchemy.engine"]) - for file in db.query(File).all(): - fileinfo = _make_backsigned_fileinfo_from_file(file) - bins_role.add_target_to_bin( - file.path, - number_of_bins=BIN_N_COUNT, - fileinfo=fileinfo, + try: + metadata_repository.delegate_targets_roles( + delegate_roles_payload, + set_expiration_for_role(config, Role.TIMESTAMP.value), + set_expiration_for_role(config, Role.SNAPSHOT.value), + ) + except FileExistsError as err: + raise click.ClickException(str(err)) + + # Delegates all BINS -> BIN_N (Hash bin prefixes) + delegate_roles_payload = dict() + delegate_roles_payload[Role.BINS.value] = list() + for bin_n_name, bin_n_hash_prefixes in hash_bins.generate(): + delegate_roles_payload[Role.BINS.value].append( + RolesPayload( + expiration=set_expiration_for_role(config, Role.BIN_N.value), + threshold=config.registry.settings[f"tuf.{Role.BIN_N.value}.threshold"], + keys=[key_service.get(Role.BIN_N.value, "private")], + delegation_role=bin_n_name, + path_hash_prefixes=bin_n_hash_prefixes, + ) ) - repository.mark_dirty(dirty_roles) - repository.writeall( - consistent_snapshot=True, - use_existing_fileinfo=True, - ) + try: + metadata_repository.delegate_targets_roles( + delegate_roles_payload, + set_expiration_for_role(config, Role.TIMESTAMP.value), + set_expiration_for_role(config, Role.SNAPSHOT.value), + ) + except FileExistsError as err: + raise click.ClickException(str(err)) diff --git a/warehouse/config.py b/warehouse/config.py index 789f1f067690..54832a892e98 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -242,7 +242,6 @@ def configure(settings=None): maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND") maybe_set_compound(settings, "tuf", "key_backend", "TUF_KEY_BACKEND") maybe_set_compound(settings, "tuf", "storage_backend", "TUF_STORAGE_BACKEND") - maybe_set_compound(settings, "tuf", "repo_backend", "TUF_REPO_BACKEND") # Pythondotorg integration settings maybe_set(settings, "pythondotorg.host", "PYTHONDOTORG_HOST", default="python.org") @@ -515,10 +514,6 @@ def configure(settings=None): # Register TUF support for package integrity config.include(".tuf") - # Serve the TUF metadata files. - # TODO: This should be routed to the TUF GCS bucket. - config.add_static_view("tuf", "warehouse:tuf/dist/metadata.staged/") - # Configure redirection support config.include(".redirects") diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index ad633bb3d12d..35917ddbc0e7 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -61,7 +61,7 @@ Release, Role, ) -from warehouse.tuf.interfaces import IRepositoryService +from warehouse.packaging.tasks import update_bigquery_release_files from warehouse.utils import http, readme ONE_MB = 1 * 1024 * 1024 @@ -1461,8 +1461,56 @@ def file_upload(request): }, ) - repository = request.find_service(IRepositoryService) - repository.add_target(file_) + # We are flushing the database requests so that we + # can access the server default values when initiating celery + # tasks. + request.db.flush() + + # Push updates to BigQuery + dist_metadata = { + "metadata_version": form["metadata_version"].data, + "name": form["name"].data, + "version": form["version"].data, + "summary": form["summary"].data, + "description": form["description"].data, + "author": form["author"].data, + "description_content_type": form["description_content_type"].data, + "author_email": form["author_email"].data, + "maintainer": form["maintainer"].data, + "maintainer_email": form["maintainer_email"].data, + "license": form["license"].data, + "keywords": form["keywords"].data, + "classifiers": form["classifiers"].data, + "platform": form["platform"].data, + "home_page": form["home_page"].data, + "download_url": form["download_url"].data, + "requires_python": form["requires_python"].data, + "pyversion": form["pyversion"].data, + "filetype": form["filetype"].data, + "comment": form["comment"].data, + "requires": form["requires"].data, + "provides": form["provides"].data, + "obsoletes": form["obsoletes"].data, + "requires_dist": form["requires_dist"].data, + "provides_dist": form["provides_dist"].data, + "obsoletes_dist": form["obsoletes_dist"].data, + "requires_external": form["requires_external"].data, + "project_urls": form["project_urls"].data, + "filename": file_data.filename, + "python_version": file_data.python_version, + "packagetype": file_data.packagetype, + "comment_text": file_data.comment_text, + "size": file_data.size, + "has_signature": file_data.has_signature, + "md5_digest": file_data.md5_digest, + "sha256_digest": file_data.sha256_digest, + "blake2_256_digest": file_data.blake2_256_digest, + "path": file_data.path, + "uploaded_via": file_data.uploaded_via, + "upload_time": file_data.upload_time, + } + if not request.registry.settings.get("warehouse.release_files_table") is None: + request.task(update_bigquery_release_files).delay(dist_metadata) # Log a successful upload metrics.increment("warehouse.upload.ok", tags=[f"filetype:{form.filetype.data}"]) diff --git a/warehouse/routes.py b/warehouse/routes.py index e9d6006af7ed..4d05b6db692c 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -17,7 +17,7 @@ def includeme(config): # Forklift is properly split out into it's own project. warehouse = config.get_settings().get("warehouse.domain") files_url = config.get_settings()["files.url"] - + metadata_url = config.get_settings()["tuf.url"] # Simple Route for health checks. config.add_route("health", "/_health/") @@ -324,6 +324,7 @@ def includeme(config): domain=warehouse, ) config.add_route("packaging.file", files_url) + config.add_route("tuf.metadata", metadata_url) # SES Webhooks config.add_route("ses.hook", "/_/ses-hook/", domain=warehouse) @@ -441,6 +442,7 @@ def includeme(config): ) config.add_redirect("/pypi/", "/", domain=warehouse) config.add_redirect("/packages/{path:.*}", files_url, domain=warehouse) + config.add_redirect("/metadata/{path:.*}", metadata_url, domain=warehouse) # Legacy Action Redirects config.add_pypi_action_redirect("rss", "/rss/updates.xml", domain=warehouse) diff --git a/warehouse/tuf/README.md b/warehouse/tuf/README.md new file mode 100644 index 000000000000..9a039834ae4e --- /dev/null +++ b/warehouse/tuf/README.md @@ -0,0 +1,260 @@ +# General TUF Warehouse implementation Notes + +## Current Warehouse and tools (pip, twine, WebUI) flow investigation + +### twine + 1. post resquest ``/simple/{project}`` and file name to Warehouse + 2. Warehouse proceed with validations + 3. Warehouse uses the ``forklift.legacy.file_upload()`` and writes in the + ``db`` and ``Storage[/packages/{blakeb_256/XX/YY}/{filename}]`` + + +### PyPI WebUI + + 1. from the ``manage.views.ManageProjectRelease()`` request to remove the a + release version using the ``utils.project.remove_project()`` + 2. The file is deleted from ``db``, but not from the `Storage[/packages]` + +### PIP +Using diferent commands ``pip `` + +#### index + 1. Request ``/simple/{project}`` + 2. Warehouse render dynamically the index + ``legacy.api.simple_detail()`` -> ``packaging.utils.simple_details()`` + if the project exists. + +#### download + 1. Call ``pip index`` + 2. Look for the latest version in the simpleindex and request from + ``Storage[/packages/{blakeb_256/XX/YY}/{filename}]`` + +#### install + 1. Call ``pip index`` + 2. Call ``pip install`` + 3. Look into the dependencies + 4. loop for the dependencies + ```mermaid + sequenceDiagram + participant pip + participant warehouse + pip->>warehouse: simple/ + warehouse-->>pip: 404, simple index not found + warehouse->>pip: 200, simple index + loop look in the index + pip->>pip: Get latest version or specific version + end + pip->>warehouse: Get specific version /packages/{blake2b_256/XX/YY}/ + warehouse-->pip: 404, not found + warehouse->>pip: 200, + pip->>pip: Looking for dependencies dependencies + ``` + + +## General flows on Warehouse +```mermaid + flowchart TD + + subgraph pip["pip "] + download + index + install + end + PyPI[PyPI WebUI] + twine + + subgraph warehouse + request["request /simple/{project} dynamic (transversal)"] + subgraph forklift + legacy.file_upload["legacy.file_upload()"] + end + subgraph legacy + api.simple_detail["api.simple_detail()"] + end + subgraph manage + views.ManageProjectRelease + end + subgraph utils + project.remove_project["project.remove_project()"] + end + subgraph packaging + utils._simple_detail["utils._simple_detail()"] + utils.render_simple_index["utils.render_simple_index()"] + end + end + + db[(Database)] + simple[("[local, SaaS]\n/simple/{project}/index.html\n/simple//.html")] + packages[("[local, SaaS]\n/packages/{blake2b_256/XX/YY}/")] + + + download--1-->request + download--2-->packages + install--1-->request + install--2-->packages + index-->request + twine-->request + PyPI-->views.ManageProjectRelease + request-->legacy.file_upload + views.ManageProjectRelease-->project.remove_project + legacy.file_upload--->db + legacy.file_upload--->packages + project.remove_project-->db + request-->api.simple_detail + api.simple_detail-->utils._simple_detail + utils.render_simple_index-.->simple + + + linkStyle 0,2,4,12,13 stroke:blue; + linkStyle 1,3 stroke:green; + linkStyle 5,7,9,10 stroke:yellow; + linkStyle 6,8,11 stroke:red; + style utils.render_simple_index fill:purple + style db fill:black,stroke:grey + style packages fill:black,stroke:grey + style simple fill:purple,stroke:grey +``` + +- Recently was merge [PR 458](https://github.com/pypa/warehouse/pull/8586), that +enables the persistent index for Simple Details. + +## TUF WIP + +The current work on Warehouse is a refactoring from the +[Draft PR](https://github.com/pypa/warehouse/pull/7488) using the old +``python-tuf`` repository tools. + +The mentioned repository tools are no longer available on the latest version. + +The idea is to start implementing small PRs to evaluate and progress. + +### Current status + +- Implemented the new repository tool using the new python-tuf Metadata API. + + Repository tool Design overview: + - The repository ``warehouse.tuf.repository.MetadataRepository`` management + tool was created standalone, not as ``zope.interface.Interface``. + + The reason is that if a better Metadata Repository management implementation + is available in the future, it could be replaced easily. + + Also, the ``MetadataRepository`` does not need to support different + implementations such as SaaS once the ``StorageService`` is enough. + + - The ``IStorageService`` was used in the + ``warehouse.tuf.repository.MetadataRepository`` as backend storage. + + - Should the ``warehouse.utils.repository_*`` be implemented as a + ``RepositoryService``? + I didn't implement it for the simple reason that it does not need to support + different implementations such as SaaS. + + ```mermaid + classDiagram + direction LR + class MetadataRepository { + <> + +storage_backend + +key_backend + initialize() + load_role() + bump_role_version() + timestamp_bump_version() + snapshot_bump_version() + snapshot_update_meta() + delegate_targets_roles() + add_targets() + } + class `tuf.interfaces` { + zope.interface.Interface + IKeyService(Interface) + IStorageService(Interface) + } + class `tuf.services` { + LocalKeyService(IKeyService) + LocalStorageService(IStorageService) + ....() + } + class `tuf.utils` { + repository_bump_snapshot() + repository_bump_bins_ns() + repository_add_target() + } + class `tuf.tasks` { + bump_snapshot() + bump_bin_ns() + add_targets() + } + + class `cli.tuf`{ + dev keypairs + dev new-repo + dev add-targets + admin delegate-targets + admin bump-snapshot + admin bump-bin-ns + } + + `tuf.utils` --* MetadataRepository + `tuf.utils` --* `tuf.services` + `tuf.services` <|-- `tuf.interfaces` + `tuf.tasks` --* `tuf.utils` + `cli.tuf` --* `tuf.utils` + warehouse -- `cli.tuf` + ``` + +- Implemented Services/Interfaces for Storage and Storage Keys +- Implemented TUF CLI + - Development Commands + - Generate Key Pairs for the roles + - Initialize the TUF Repository (keys and metadata) for Development + - Add targets files (Packages from Database) + - Administrative Commands + - Generate the delegations Roles and Hashed bin roles (bins and bin-n) + - Bump snapshot + - Bump delegate bin-n hashed bin targets roles +- Tasks + - Update Snapshot role metadata + - Update delegated hash bin roles (bin-n) and their delegations + +It's is possible use the new ``python-tuf`` client (ngclient) to download the +target. + + + +### Next steps: + +- [ ] Polish the new Warehouse metadata repository tool +- [ ] Publish the new Draft PR + + Goals: + - Limited for enabling tuf Metadata on development environment + - Intent to validate the Metadata Repository tool + +- [ ] PR to implement the simple detail index to the Metadata + +- [ ] PR to implement the TUF in the Warehouse request flow + - upload target file + - delete target file + - tasks for refreshing indexes/projects + + +## Using the Warehouse development environment for TUF + +Follow the official Warehouse until ``make initdb`` + +```shell +$ make inittuf +``` + +The metadata is available at http://localhost:9001/metadata/ + +You can also upload a file using the Warehouse and add the targets using CLI +- Create a user [using Web UI](https://warehouse.pypa.io/development/getting-started.html#viewing-warehouse-in-a-browser) +- Validate the [email](https://warehouse.pypa.io/development/email.html) +- Upload file using ``twine`` + +```shell +docker-compose run --rm web python -m warehouse tuf dev add-targets +``` diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 8a1878f0e2c4..ca7d02cf23d4 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -12,7 +12,8 @@ from celery.schedules import crontab -from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService +from warehouse.tuf.interfaces import IKeyService, IStorageService +from warehouse.tuf.repository import SPEC_VERSION from warehouse.tuf.tasks import bump_bin_ns, bump_snapshot @@ -38,11 +39,13 @@ def includeme(config): # An amended version of the PEP should be published, at which point # this note can be removed. "tuf.bin-n.expiry": 604800, - "tuf.spec_version": "1.0.0", + "tuf.spec_version": SPEC_VERSION, } ) - key_service_class = config.maybe_dotted(config.registry.settings["tuf.key_backend"]) + key_service_class = config.maybe_dotted( + config.registry.settings["tuf.key_backend"] + ) config.register_service_factory(key_service_class.create_service, IKeyService) storage_service_class = config.maybe_dotted( @@ -52,13 +55,6 @@ def includeme(config): storage_service_class.create_service, IStorageService ) - repo_service_class = config.maybe_dotted( - config.registry.settings["tuf.repo_backend"] - ) - config.register_service_factory( - repo_service_class.create_service, IRepositoryService - ) - # Per PEP 458: The snapshot and timestamp metadata expire every 24 hours. # We conservatively bump them every 6 hours. # Note that bumping the snapshot causes us to bump the timestamp, so we diff --git a/warehouse/tuf/constants.py b/warehouse/tuf/constants.py index 0f42a1162629..3ed4c8fbfa05 100644 --- a/warehouse/tuf/constants.py +++ b/warehouse/tuf/constants.py @@ -23,13 +23,6 @@ class Role(enum.Enum): BIN_N: str = "bin-n" -TOPLEVEL_ROLES = [ - Role.ROOT.value, - Role.SNAPSHOT.value, - Role.TARGETS.value, - Role.TIMESTAMP.value, -] - HASH_ALGORITHM = "blake2b" TUF_REPO_LOCK = "tuf-repo" diff --git a/warehouse/tuf/hash_bins.py b/warehouse/tuf/hash_bins.py new file mode 100644 index 000000000000..58e4b80d0905 --- /dev/null +++ b/warehouse/tuf/hash_bins.py @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib + +from typing import Iterator, List, Tuple + + +class HashBins: + """ + Hash Bins management + + This code is inspired on python-tuf repo examples for hash bins + """ + def __init__(self, number_of_bins: int) -> None: + """ + Hash Bins + + Args: + number_of_bins: bumber of bins + """ + self.number_of_bins = number_of_bins + # The prefix length is the number of digits in the hexadecimal representation + # (see 'x' in Python Format Specification) of the number of bins minus one + # (counting starts at zero), i.e. ... + self.prefix_len = len(f"{(self.number_of_bins - 1):x}") # ... 2. + # Compared to decimal, hexadecimal numbers can express higher numbers + # with fewer digits and thus further decrease metadata sizes. With the + # above prefix length of 2 we can represent at most 256 prefixes, i.e. + # 00, 01, ..., ff. + self.number_of_prefixes = 16**self.prefix_len + # If the number of bins is a power of two, hash prefixes are evenly + # distributed over all bins, which allows to calculate the uniform size + # of 8, where each bin is responsible for a range of 8 prefixes, i.e. + # 00-07, 08-0f, ..., f8-ff. + self.bin_size = self.number_of_prefixes // self.number_of_bins + + def _bin_name(self, low: int, high: int) -> str: + """ + Generates a bin name according to the hash prefixes the bin serves. + + The name is either a single hash prefix for bin size 1, or a range of hash + prefixes otherwise. The prefix length is needed to zero-left-pad the + hex representation of the hash prefix for uniform bin name lengths. + """ + if low == high: + return f"{low:0{self.prefix_len}x}" + + return f"{low:0{self.prefix_len}x}-{high:0{self.prefix_len}x}" + + def generate(self) -> Iterator[Tuple[str, List[str]]]: + """Returns generator for bin names and hash prefixes per bin.""" + # Iterate over the total number of hash prefixes in 'bin size'-steps to + # generate bin names and a list of hash prefixes served by each bin. + for low in range(0, self.number_of_prefixes, self.bin_size): + high = low + self.bin_size - 1 + bin_name = self._bin_name(low, high) + hash_prefixes = [] + for prefix in range(low, low + self.bin_size): + hash_prefixes.append(f"{prefix:0{self.prefix_len}x}") + + yield bin_name, hash_prefixes + + def get_delegate(self, file_path: str) -> str: + """ + Gets the delegated role name bin based on the target file path. + + Args: + file_path + + Returns: + bin name low-high + """ + hasher = hashlib.sha256() + hasher.update(file_path.encode("utf-8")) + target_name_hash = hasher.hexdigest() + prefix = int(target_name_hash[: self.prefix_len], 16) + low = prefix - (prefix % self.bin_size) + high = low + self.bin_size - 1 + return self._bin_name(low, high) diff --git a/warehouse/tuf/interfaces.py b/warehouse/tuf/interfaces.py index 7a4e4915abab..f0e9dc297e98 100644 --- a/warehouse/tuf/interfaces.py +++ b/warehouse/tuf/interfaces.py @@ -20,15 +20,8 @@ def create_service(context, request): created. """ - def pubkeys_for_role(rolename): - """ - Return a list of (TUF-formatted) public keys for the given TUF role. - """ - - def privkeys_for_role(rolename): - """ - Return a list of (TUF-formatted) private keys for the given TUF role. - """ + def get(rolename, key_type): + """Return a key from specific rolename""" class IStorageService(Interface): @@ -43,26 +36,5 @@ def get_backend(): Return an implementation of `securesystemslib.storage.StorageBackendInterface`. """ - -class IRepositoryService(Interface): - def create_service(context, request): - """ - Create the service, given the context and request for which it is being - created. - """ - - def load_repository(): - """ - Return a TUF Repository object for direct manipulation of the underlying - repository. - - NOTE: The Repository object returned from this method cannot be manipulated - safely by multiple tasks or threads, especially. It should only be used during - TUF initialization or offline maintenance tasks. - """ - - def add_target(file, backsigned=False): - """ - Given a warehouse.packaging.models.File, add it to the TUF - repository. - """ + def get(rolename): + """Return a key from specific rolename""" diff --git a/warehouse/tuf/repository.py b/warehouse/tuf/repository.py new file mode 100644 index 000000000000..b4dbf423b324 --- /dev/null +++ b/warehouse/tuf/repository.py @@ -0,0 +1,504 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Dict, List, Optional + +from securesystemslib.exceptions import StorageError +from securesystemslib.signer import SSlibSigner +from tuf.api.metadata import ( + SPECIFICATION_VERSION, + TOP_LEVEL_ROLE_NAMES, + DelegatedRole, + Delegations, + Key, + Metadata, + MetaFile, + Role, + Root, + Snapshot, + StorageBackendInterface, + TargetFile, + Targets, + Timestamp, +) +from tuf.api.serialization.json import JSONSerializer + +from warehouse.tuf.constants import Role as RoleWarehouse +from warehouse.tuf.interfaces import IKeyService + +SPEC_VERSION = ".".join(SPECIFICATION_VERSION) + + +@dataclass +class TUFRepositoryFiles: + filename: str + data: Metadata + + +@dataclass +class RolesPayload: + expiration: datetime + threshold: int + keys: List[Dict[str, Any]] + delegation_role: str = None + paths: List[str] = None + path_hash_prefixes: List[str] = None + + +@dataclass +class TargetsPayload: + fileinfo: str + path: str + + +class MetadataRepository: + def __init__( + self, + storage_backend: StorageBackendInterface, + key_backend: IKeyService, + ): + """ + TUF Metadata Repository Management + + Args: + storage_backend: Storage Backend + key_backend: Key Sorage Backend + """ + self.storage_backend: StorageBackendInterface = storage_backend + self.key_backend: IKeyService = key_backend + self._is_initialized: bool = self._check_is_initialized() + + @property + def is_initialized(self) -> bool: + """State of repository + + Returns: + bool + """ + return self._is_initialized + + def _create_delegated_targets_roles( + self, + delegator_metadata: Metadata, + snapshot_metadata: Metadata[Snapshot], + delegate_role_parameters: List[RolesPayload], + ) -> Metadata[Snapshot]: + """ + Creates delegated targets roles based on delegator metadata and a + list of roles parameters (``RolesPayload``). + + 1. Creates the DelegatedRole Object + 2. Adds to the Delegator Metadata Delegations + 3. Generates the new Delegated Role Metadata + 4. Add the keys and signs (using keys from Key backend) + 5. stores it in the repository storage (using the storage backend) + + Args + delegator_metadata: Delegator Metadata + snapshot_metadata: Snapshot Metadata + delegated_role_parameters: List of Roles Parameters + + Returns: + The updated Snapshot Metadata + ``tuf.api.metadata.Metadata`` + """ + + for role_parameter in delegate_role_parameters: + rolename = role_parameter.delegation_role + try: + if self.load_role(rolename): + raise FileExistsError(f"Role {rolename} already exists.") + except StorageError: + pass + + delegated_role = DelegatedRole( + name=rolename, + keyids=[key["keyid"] for key in role_parameter.keys], + threshold=role_parameter.threshold, + terminating=None, + paths=role_parameter.paths, + path_hash_prefixes=role_parameter.path_hash_prefixes, + ) + + if delegator_metadata.signed.delegations is None: + delegation = self._build_delegations( + rolename, delegated_role, role_parameter.keys + ) + delegator_metadata.signed.delegations = delegation + else: + delegator_metadata.signed.delegations.roles[rolename] = delegated_role + + targets = Targets(1, SPEC_VERSION, role_parameter.expiration, {}, None) + role_metadata = Metadata(targets, {}) + + for key in role_parameter.keys: + delegator_metadata.signed.add_key( + rolename, Key.from_securesystemslib_key(key) + ) + role_metadata.sign(SSlibSigner(key)) + + self._store(rolename, role_metadata) + self.snapshot_update_meta(rolename, role_metadata.signed.version) + + return snapshot_metadata + + def _check_is_initialized(self) -> None: + """ + Check if Repository metadata is initialized based on the top-level + metadata files in the storage backend and update the ``is_initialized`` + state. + """ + try: + if any( + role + for role in TOP_LEVEL_ROLE_NAMES + if isinstance(self.load_role(role), Metadata) + ): + self._is_initialized = True + except StorageError: + pass + + def _filename(self, rolename: str, version: int) -> str: + """Build the filename based in the role name and version. + + Args: + rolename: Role name + version: Role version + + Returns: + File name + """ + if rolename == Timestamp.type: + filename = f"{rolename}.json" + else: + filename = f"{version}.{rolename}.json" + + return filename + + def _store(self, rolename: str, metadata: Metadata) -> None: + """ + Stores (writes) the Metadata to the Storage using the Storage Backend. + + Args: + rolename: Role name + metadata: Role Metadata + """ + filename = self._filename(rolename, metadata.signed.version) + metadata.to_file(filename, JSONSerializer(), self.storage_backend) + + def _build_delegations( + self, rolename: str, delegated_role: DelegatedRole, keys: List[Dict[str, Any]] + ) -> Delegations: + """ + Builds the Delegations container object storing keys and DelegatedRole + information. + + Args: + rolename: Role name + delegated_role: Delegated Role + keys: List of Keys Dictionary + + Returns + The Delegations container object + ``tuf.api.metadata.Delegations`` + """ + return Delegations( + keys={key["keyid"]: Key.from_securesystemslib_key(key) for key in keys}, + roles={rolename: delegated_role}, + ) + + def initialize( + self, payload: Dict[str, RolesPayload], store: Optional[bool] + ) -> Dict[str, Metadata]: + """ + Initialize the TUF repository from scratch, including a brand new root. + + Args: + payload: Dictionary of the role name and the role parameters + store: After initialization stores the metadata files + + Returns: + Dictionary with key as role name and value as Metadata + ``Dict[str, Metadata]`` + """ + + self._check_is_initialized() + + top_level_roles_metadata = dict() + if self.is_initialized: + raise FileExistsError("Metadata already exists in the Storage Service") + + targets = Targets(1, SPEC_VERSION, payload[Targets.type].expiration, {}, None) + targets_metadata = Metadata(targets, {}) + top_level_roles_metadata[Targets.type] = targets_metadata + + meta = {"targets.json": MetaFile(targets.version)} + snapshot = Snapshot(1, SPEC_VERSION, payload[Snapshot.type].expiration, meta) + snapshot_metadata = Metadata(snapshot, {}) + top_level_roles_metadata[Snapshot.type] = snapshot_metadata + + snapshot_meta = MetaFile(snapshot.version) + timestamp = Timestamp( + 1, SPEC_VERSION, payload[Timestamp.type].expiration, snapshot_meta + ) + timestamp_metadata = Metadata(timestamp, {}) + top_level_roles_metadata[Timestamp.type] = timestamp_metadata + + roles = { + role_name: Role([], payload[role_name].threshold) + for role_name in TOP_LEVEL_ROLE_NAMES + } + root = Root(1, SPEC_VERSION, payload[Root.type].expiration, {}, roles, True) + + # Sign all top level roles metadata + signers = dict() + for role in TOP_LEVEL_ROLE_NAMES: + if payload[role].threshold < len(payload[role].keys): + raise ValueError( + f"Role {role} has missing Key(s) " + f"to match to defined threshold {payload[role].threshold}." + ) + for key in payload[role].keys: + root.add_key(role, Key.from_securesystemslib_key(key)) + signers[role] = {key["keyid"]: SSlibSigner(key)} + + root_metadata = Metadata(root, {}) + top_level_roles_metadata[Root.type] = root_metadata + + for role in signers: + for signer in signers[role].values(): + top_level_roles_metadata[role].sign(signer, append=True) + + if store: + self._store(role, top_level_roles_metadata[role]) + + self._is_initialized = True + return top_level_roles_metadata + + def load_role(self, rolename: str) -> Metadata: + """ + Loads a specific metadata by role name. + + Args: + rolename: Role name + + Returns: + Role Metadata + ``tuf.api.metadata.Metadata`` + """ + return Metadata.from_file(rolename, None, self.storage_backend) + + def delegate_targets_roles( + self, + payload: Dict[str, List[RolesPayload]], + timestamp_expires: datetime, + snapshot_expires: datetime, + ) -> None: + """ + Delegates a list of targets roles. + It will create the new role based in the parameters, sign and store, + updating the Delegator role, Snapshot and Timestamp roles. + + Args: + payload: Dictionary of delegators with the delegated role parameters + as RolePayload + timestamp_expires: New Timestamp expiration datetime + snapshot_expires: New Snapshot expiration datetime + """ + + snapshot_metadata = self.load_role(Snapshot.type) + for delegator, delegate_role_parameters in payload.items(): + delegator_metadata = self.load_role(delegator) + snapshot_metadata = self._create_delegated_targets_roles( + delegator_metadata, snapshot_metadata, delegate_role_parameters + ) + snapshot_metadata = self.bump_role_version( + rolename=delegator, + role_metadata=delegator_metadata, + role_expires=delegator_metadata.signed.expires, + snapshot_metadata=snapshot_metadata, + key_rolename=None, + store=True, + ) + + snapshot_metadata = self.snapshot_bump_version( + snapshot_expires, snapshot_metadata, True + ) + self.timestamp_bump_version( + snapshot_metadata.signed.version, timestamp_expires, True + ) + + def bump_role_version( + self, + rolename: str, + role_metadata: Metadata, + role_expires: datetime, + snapshot_metadata: Metadata[Snapshot], + key_rolename: Optional[str] = None, + store: Optional[bool] = False, + ) -> Metadata[Snapshot]: + """ + Bump an specific Role Metadata (ex: delegated target role) version, + optionaly stores it in the repository storage and uppdating the Snapshot + role meta. + + Args: + rolename: Specific Role name to have the version bumped + role_metadata: Specific Role Metadata + role_expires: expiration datetime + snapshot_metadata: Snapshot Metadata + key_rolename: Different signing Key role name + store: Optional, Default False, stores in the repository storage + Returns: + Updated Snapshot Metadata + ``Metadata[Snapshot]`` + """ + if key_rolename: + key_rolename = key_rolename + else: + key_rolename = rolename + role_metadata.signed.expires = role_expires + role_metadata.signed.version += 1 + role_metadata.sign(SSlibSigner(self.key_backend.get(key_rolename, "private"))) + + if store: + self._store(rolename, role_metadata) + + return self.snapshot_update_meta( + rolename, role_metadata.signed.version, snapshot_metadata + ) + + def timestamp_bump_version( + self, + snapshot_version: int, + timestamp_expires: datetime, + store: bool = False, + ) -> Metadata[Timestamp]: + """ + Bump the Timestamp Role Metadata updating the Snapshot version, and + optionally stores it in the repository storage. + + Args: + snapshot_version: Snapshot version to be updated on Timestamp role + timestamp_expires: New Timestamp expiration datetime + store: Optional, Default False, stores in the repository storage + + Returns: + Timestamp Metadata + ``Metadata[Timestamp]`` + """ + timestamp_metadata = self.load_role(Timestamp.type) + timestamp_metadata.signed.version += 1 + timestamp_metadata.signed.expires = timestamp_expires + timestamp_metadata.signed.snapshot_meta = MetaFile(version=snapshot_version) + timestamp_metadata.sign( + SSlibSigner(self.key_backend.get(Timestamp.type, "private")) + ) + + if store: + self._store(Timestamp.type, timestamp_metadata) + + return timestamp_metadata + + def snapshot_bump_version( + self, + snapshot_expires: datetime, + snapshot_metadata: Optional[Metadata[Snapshot]] = None, + store: Optional[bool] = False, + ) -> Metadata[Snapshot]: + """ + Bump Snapshot Role Metadata version and optionally stores it. + + Args: + snapshot_expires: New Snapshot expiration datetime + snapshot_metadata: Optional, specific Snapshot Metadata + store: Optional, Default False, stores in the repository storage + + Returns: + Snapshot metadata + ``Metadata[Snapshot]`` + """ + if snapshot_metadata is None: + snapshot_metadata = self.load_role(Snapshot.type) + + snapshot_metadata.signed.version += 1 + snapshot_metadata.signed.expires = snapshot_expires + snapshot_metadata.sign( + SSlibSigner(self.key_backend.get(Snapshot.type, "private")) + ) + + if store is True: + self._store(Snapshot.type, snapshot_metadata) + + return snapshot_metadata + + def snapshot_update_meta( + self, + meta_role_name: str, + meta_role_version: int, + snapshot_metadata: Optional[Metadata[Snapshot]] = None, + ) -> Metadata[Snapshot]: + """ + Update the Snapshot Metadata Meta. + + Args: + meta_role_name: role name + meta_role_version: role version + snapshot_metadata: Optional, specific Snapshot Metadata + + Return: + Snapshot metadata with updated Meta + ``Metadata[Snapshot]`` + """ + if snapshot_metadata is None: + snapshot_metadata = self.load_role(Snapshot.type) + + snapshot_metadata.signed.meta[f"{meta_role_name}.json"] = MetaFile( + version=meta_role_version + ) + + return snapshot_metadata + + def add_targets( + self, + payload: Dict[str, List[TargetsPayload]], + timestamp_expires: datetime, + snapshot_expires: datetime, + ) -> None: + """ + Add a list of targets from a payload to the Role Metadata, bumps the + role Metadata version, stores, bumps Timestamp and + Snapshot Metadata version and store them. + + Args: + payload: Dictionary with role names and list of ``TargetPayload`` + timestamp_expires: New Timestamp expiration datetime + snapshot_expires: New Snapshot expiration datetime + """ + snapshot_metadata = self.load_role(Snapshot.type) + + for rolename, targets in payload.items(): + role_metadata = self.load_role(rolename) + for target in targets: + target_file = TargetFile.from_dict(target.fileinfo, target.path) + role_metadata.signed.targets[target.path] = target_file + + role_metadata.signed.version += 1 + role_metadata.sign( + SSlibSigner(self.key_backend.get(RoleWarehouse.BIN_N.value, "private")) + ) + self._store(rolename, role_metadata) + snapshot_metadata = self.bump_role_version( + rolename=rolename, + role_metadata=role_metadata, + role_expires=role_metadata.signed.expires, + snapshot_metadata=snapshot_metadata, + key_rolename=RoleWarehouse.BIN_N.value, + store=True, + ) + + snapshot_metadata = self.snapshot_bump_version( + snapshot_expires, snapshot_metadata, True + ) + self.timestamp_bump_version( + snapshot_metadata.signed.version, timestamp_expires, True + ) diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py index 3b39951d2b64..d1dc8e1cb79b 100644 --- a/warehouse/tuf/services.py +++ b/warehouse/tuf/services.py @@ -11,15 +11,24 @@ # limitations under the License. +import glob import os.path +import shutil import warnings -from tuf import repository_tool +from contextlib import contextmanager + +from securesystemslib.exceptions import StorageError +from securesystemslib.interface import ( + import_ed25519_privatekey_from_file, + import_ed25519_publickey_from_file, +) from zope.interface import implementer -from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService +from warehouse.tuf.constants import Role +from warehouse.tuf.interfaces import IKeyService, IStorageService from warehouse.tuf.tasks import add_target -from warehouse.tuf.utils import GCSBackend, LocalBackend, make_fileinfo +from warehouse.tuf.utils import GCSBackend, make_fileinfo class InsecureKeyWarning(UserWarning): @@ -42,48 +51,31 @@ def __init__(self, key_path, request): def create_service(cls, context, request): return cls(request.registry.settings["tuf.key.path"], request) - def pubkeys_for_role(self, rolename): - pubkey_path = os.path.join(self._key_path, f"tuf.{rolename}.pub") - return [repository_tool.import_ed25519_publickey_from_file(pubkey_path)] - - def privkeys_for_role(self, rolename): - privkey_path = os.path.join(self._key_path, f"tuf.{rolename}") - return [ - repository_tool.import_ed25519_privatekey_from_file( - privkey_path, - password=self._request.registry.settings[f"tuf.{rolename}.secret"], - ) - ] + def get(self, rolename, key_type): + """ + Gets the Key based on role name and key type + Args: + rolename: role name + key_type: 'private' or 'public' -@implementer(IStorageService) -class LocalStorageService: - def __init__(self, request): - self._store = LocalBackend(request) - - @classmethod - def create_service(cls, context, request): - return cls(request) + """ + if key_type == "private": + privkey_path = os.path.join(self._key_path, f"{rolename}") + key_sslib = import_ed25519_privatekey_from_file( + privkey_path, self._request.registry.settings[f"tuf.{rolename}.secret"] + ) + elif key_type == "public": + pubkey_path = os.path.join(self._key_path, f"{rolename}.pub") + key_sslib = import_ed25519_publickey_from_file(pubkey_path) + else: + raise ValueError(f"invalid key_type '{key_type}'") - def get_backend(self): - return self._store + return key_sslib @implementer(IStorageService) -class GCSStorageService: - def __init__(self, request): - self._store = GCSBackend(request) - - @classmethod - def create_service(cls, context, request): - return cls(request) - - def get_backend(self): - return self._store - - -@implementer(IRepositoryService) -class LocalRepositoryService: +class LocalStorageService: def __init__(self, repo_path, executor): self._repo_path = repo_path self._executor = executor @@ -95,6 +87,51 @@ def create_service(cls, context, request): request.task(add_target).delay, ) + @contextmanager + def get(self, role, version=None): + if role == Role.TIMESTAMP.value: + filename = os.path.join(self._repo_path, f"{role}.json") + else: + if version is None: + filenames = glob.glob(os.path.join(self._repo_path, f"*.{role}.json")) + versions = [ + int(name.split("/")[-1].split(".", 1)[0]) for name in filenames + ] + try: + version = max(versions) + except ValueError: + version = 1 + + filename = os.path.join(self._repo_path, f"{version}.{role}.json") + if not os.path.isfile(filename): + filename = os.path.join(self._repo_path, f"{role}.json") + + file_object = None + try: + file_object = open(filename, "rb") + yield file_object + except OSError: + raise StorageError(f"Can't open {filename}") + finally: + if file_object is not None: + file_object.close() + + def put(self, file_object, filename): + file_path = os.path.join(self._repo_path, filename) + if not file_object.closed: + file_object.seek(0) + + try: + with open(file_path, "wb") as destination_file: + shutil.copyfileobj(file_object, destination_file) + destination_file.flush() + os.fsync(destination_file.fileno()) + except OSError: + raise StorageError(f"Can't write file {filename}") + + def store(self, file_object, filename): + self.put(file_object, filename) + def load_repository(self): return repository_tool.load_repository(self._repo_path) @@ -103,18 +140,14 @@ def add_target(self, file, custom=None): self._executor(file.path, fileinfo) -@implementer(IRepositoryService) -class GCSRepositoryService: - def __init__(self, executor, request): +@implementer(IStorageService) +class GCSStorageService: + def __init__(self, request): self._store = GCSBackend(request) @classmethod def create_service(cls, context, request): - return cls(request.task(add_target).delay, request) - - def load_repository(self): - return repository_tool.load_repository("tuf", storage_backend=self._store) + return cls(request) - def add_target(self, file, custom=None): - fileinfo = make_fileinfo(file, custom=custom) - self._executor(file.path, fileinfo) + def get_backend(self): + return self._store diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py index b83ef2889960..c56c7ce41a71 100644 --- a/warehouse/tuf/tasks.py +++ b/warehouse/tuf/tasks.py @@ -9,18 +9,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from datetime import timedelta - import redis -from securesystemslib.util import get_file_hashes -from tuf.api import metadata - from warehouse.tasks import task -from warehouse.tuf import utils -from warehouse.tuf.constants import HASH_ALGORITHM, TUF_REPO_LOCK, Role +from warehouse.tuf.constants import TUF_REPO_LOCK from warehouse.tuf.interfaces import IKeyService, IStorageService +from warehouse.tuf.utils import ( + repository_add_target, + repository_bump_bins_ns, + repository_bump_snapshot, +) @task(bind=True, ignore_result=True, acks_late=True) @@ -34,75 +32,10 @@ def bump_snapshot(task, request): r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) with r.lock(TUF_REPO_LOCK): - # Bumping the snapshot role involves the following steps: - # 1. First, we grab our key and storage services. We'll use the former - # for signing operations, and the latter to read and write individual - # metadata files to and from the repository without loading the entire - # repo. - # 2. Using our storage service, we fetch the timestamp metadata, which - # is always at `timestamp.json`. We load it using the `Timestamp` model - # provided by the TUF API. - # 3. Using the snapshot version stored in the current `Timestamp`, we fetch - # `{VERSION}.snapshot.json` and load it using the `Snapshot` model - # provided by the TUF API. - # 4. We call `utils.bump_metadata()` and `Snapshot.sign()` to bump - # and re-sign the current snapshot. - # 5. We call `Snapshot.to_json_file()` with `{VERSION + 1}.snapshot.json`, - # where `{VERSION + 1}` is the incremented snapshot version. - # 6. We call `Timestamp.update()` on the loaded timestamp, giving it the - # incremented snapshot version as well as the serialized length and - # BLAKE2B hash of the serialized form. - # 7. We call `utils.bump_metadata()` and `Timestamp.sign()` to bump - # and re-sign the current timestamp. - # 8. We call `Timestamp.to_json_file()`, writing to `timestamp.json`. - # - # Each of the steps is labeled below for clarity. - - # 1. Service retrieval. storage_service = request.find_service(IStorageService) key_service = request.find_service(IKeyService) - storage_backend = storage_service.get_backend() - - # 2. Timestamp retrieval and loading. - timestamp = metadata.Metadata.from_json_file("timestamp.json", storage_backend) - - # 3. Snapshot retrieval and loading. - snapshot = utils.find_snapshot(timestamp.signed, storage_backend) - - # 4. Snapshot bumping and versioning. - utils.bump_metadata( - snapshot.signed, - timedelta(seconds=request.registry.settings["tuf.snapshot.expiry"]), - ) - for key in key_service.privkeys_for_role(Role.SNAPSHOT.value): - snapshot.sign(key) - - # 5. Writing the updated snapshot back to the repository. - snapshot_filename = f"{snapshot.signed.version}.snapshot.json" - snapshot.to_json_file(snapshot_filename, storage_backend) - - # 6. Timestamp updating. - timestamp.signed.update( - snapshot.signed.version, - len(snapshot.to_json().encode()), - get_file_hashes( - snapshot_filename, - hash_algorithms=[HASH_ALGORITHM], - storage_backend=storage_backend, - ), - ) - - # 7. Timestamp bumping. - utils.bump_metadata( - timestamp.signed, - timedelta(seconds=request.registry.settings["tuf.timestamp.expiry"]), - ) - for key in key_service.privkeys_for_role(Role.TIMESTAMP.value): - timestamp.sign(key) - - # 8. Writing the updated timestamp back to the repository. - timestamp.to_json_file("timestamp.json", storage_backend) + repository_bump_snapshot(request, storage_service, key_service) @task(bind=True, ignore_result=True, acks_late=True) @@ -110,95 +43,10 @@ def bump_bin_ns(task, request): r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) with r.lock(TUF_REPO_LOCK): - # Bumping all of the delegated bin roles in the TUF repository involves - # the following steps: - # 1. Grab key and storage services. - # 2. Fetch timestamp. - # 3. Fetch snapshot using timestamp. - # 4. For each delegated target in the snapshot, fetch its current version, - # bump, re-sign, and write back to the repo. Update the snapshot to - # match the bumped version. - # 5. Bump and re-sign the snapshot. - # 6. Write the snapshot back. - # 7. Bump and re-sign the timestamp. - # 8. Write the timestamp back. - - # 1. Service retrieval. storage_service = request.find_service(IStorageService) key_service = request.find_service(IKeyService) - storage_backend = storage_service.get_backend() - - # 2. Timestamp retrieval and loading. - timestamp = metadata.Metadata.from_json_file("timestamp.json", storage_backend) - - # 3. Snapshot retrieval and loading. - snapshot = utils.find_snapshot(timestamp.signed, storage_backend) - - # Target iteration: skip the top-level targets role. - for role_name, role_info in snapshot.signed.meta.items(): - if role_name == "targets.json": - continue - - role_version = role_info["version"] - delegated_bin_filename = f"{role_version}.{role_name}" - - # Load the delegated bin. - delegated_bin = metadata.Metadata.from_json_file( - delegated_bin_filename, storage_backend - ) - - # Bump and sign the delegated bin. - utils.bump_metadata( - delegated_bin.signed, - timedelta(seconds=request.registry.settings["tuf.bin-n.expiry"]), - ) - - for key in key_service.privkeys_for_role(Role.BIN_N.value): - delegated_bin.sign(key) - - # Write-back. - delegated_bin.to_json_file(delegated_bin_filename, storage_backend) - - # Update the snapshot with this updated target's version. - # TODO: Ideally we'd use snapshot.update here, but that takes - # the role name without .json on the end. But role_name here - # has that suffix. Annoying. - snapshot.meta[role_name]["version"] = delegated_bin.signed.version - - # Bump and sign the snapshot. - utils.bump_metadata( - snapshot.signed, - timedelta(seconds=request.registry.settings["tuf.snapshot.expiry"]), - ) - - for key in key_service.privkeys_for_role(Role.SNAPSHOT.value): - snapshot.sign(key) - - # Write-back. - snapshot_filename = f"{snapshot.signed.version}.snapshot.json" - snapshot.to_json_file(snapshot_filename, storage_backend) - - # Bump and sign the timestamp. - utils.bump_metadata( - timestamp.signed, - timedelta(seconds=request.registry.settings["tuf.timestamp.expiry"]), - ) - timestamp.signed.update( - snapshot.signed.version, - len(snapshot.to_json().encode()), - get_file_hashes( - snapshot_filename, - hash_algorithms=[HASH_ALGORITHM], - storage_backend=storage_backend, - ), - ) - - for key in key_service.privkeys_for_role(Role.TIMESTAMP.value): - timestamp.sign(key) - - # Write-back. - timestamp.to_json_file("timestamp.json", storage_backend) + repository_bump_bins_ns(request, storage_service, key_service) @task(bind=True, ignore_result=True, acks_late=True) @@ -206,118 +54,7 @@ def add_target(task, request, filepath, fileinfo): r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) with r.lock(TUF_REPO_LOCK): - # Adding a target to the TUF repository involves the following steps: - # 1. First, we grab our key and storage services. We'll use the former - # for signing operations, and the latter to read and write individual - # metadata files to and from the repository without loading the entire - # repo. - # 2. Using our storage service, we fetch the timestamp metadata, which - # is always at `timestamp.json`. We load it using the `Timestamp` model - # provided by the TUF API. - # 3. Using the snapshot version stored in the current `Timestamp`, we fetch - # `{VERSION}.snapshot.json` and load it using the `Snapshot` model - # provided by the TUF API. - # 4. Using the target's name (`filepath`), we determine the name of the - # delegated bin that it'll end up in. We use that delegated bin name to - # index into the `Snapshot` model and get the current version for that bin. - # Then, we fetch `{VERSION}.{BIN}.json` and load it using the `Targets` - # model provided by the TUF API. - # 5. We update the delegated bin, bumping its version, expiration, and - # adding our new target to it. - # 6. We call `Targets.sign()` on the loaded bin, giving it the bin-n - # signing key from our key service. - # 7. We call `Targets.to_json_file()` with `{VERSION + 1}.{BIN}.json` - # as the filepath, where `{VERSION + 1}` is the incremented version - # of the previous delegated bin version. - # 8. We update the snapshot, bumping its version, expiration, and giving - # it our new delegated bin version. - # 9. We call `Snapshot.sign()` on the loaded snapshot, giving it the - # snapshot signing key from our key service. - # 10. We call `Snapshot.to_json_file()` with `{VERSION + 1}.snapshot.json`, - # where `{VERSION + 1}` is the incremented version of the previous - # snapshot version. - # 11. We update the timestamp, bumping its version, expiration, and giving - # it our new snapshot version and integrity information. - # 12. We call `Timestamp.sign()` on the loaded timestamp, giving it the - # timestamp signing key from our key service. - # 13. We call `Timestamp.to_json_file()`, writing to `timestamp.json`. - # - # Each of the steps is labeled below for clarity. - - # 1. Service retrieval. storage_service = request.find_service(IStorageService) key_service = request.find_service(IKeyService) - storage_backend = storage_service.get_backend() - - # 2. Timestamp retrieval and loading. - timestamp = metadata.Metadata.from_json_file("timestamp.json", storage_backend) - - # 3. Snapshot retrieval and loading. - snapshot = utils.find_snapshot(timestamp.signed, storage_backend) - - # 4. Delegated bin retrieval and loading. - delegated_bin_name, delegated_bin = utils.find_delegated_bin( - filepath, snapshot.signed, storage_backend - ) - - # 5. Updating the delegated bin. - utils.bump_metadata( - delegated_bin.signed, - timedelta(seconds=request.registry.settings["tuf.bin-n.expiry"]), - ) - delegated_bin.signed.update(filepath, fileinfo) - - # 6. Signing the updated delegated bin metadata. - for key in key_service.privkeys_for_role(Role.BIN_N.value): - delegated_bin.sign(key) - - # 7. Writing the updated delegated bin back to the TUF repository. - delegated_bin.to_json_file( - f"{delegated_bin.signed.version}.{delegated_bin_name}.json", storage_backend - ) - - # 8. Updating the snapshot. - # TODO(ww): Fill in length and hashes? - utils.bump_metadata( - snapshot.signed, - timedelta(seconds=request.registry.settings["tuf.snapshot.expiry"]), - ) - snapshot.signed.update(f"{delegated_bin_name}.json", delegated_bin.signed.version) - - # 9. Signing the updated snapshot metadata. - for key in key_service.privkeys_for_role(Role.SNAPSHOT.value): - snapshot.sign(key) - - # 10. Writing the updated snapshot back to the TUF repository. - snapshot_filename = f"{snapshot.signed.version}.snapshot.json" - snapshot.to_json_file( - snapshot_filename, - storage_backend, - ) - - # 11. Updating the timestamp. - # NOTE(ww): Calling get_file_hashes here causes us to round-trip - # through the object store just to compute our snapshot's hash. - # Maybe add a function to securesystemslib that does the digest - # calculation on a string/bytes. - utils.bump_metadata( - timestamp.signed, - timedelta(seconds=request.registry.settings["tuf.timestamp.expiry"]), - ) - timestamp.signed.update( - snapshot.signed.version, - len(snapshot.to_json().encode()), - get_file_hashes( - snapshot_filename, - hash_algorithms=[HASH_ALGORITHM], - storage_backend=storage_backend, - ), - ) - - # 12. Signing the updated timestamp metadata. - for key in key_service.privkeys_for_role(Role.TIMESTAMP.value): - timestamp.sign(key) - - # 13. Writing the updated timestamp back to the TUF repository. - timestamp.to_json_file("timestamp.json", storage_backend) + repository_add_target(request, fileinfo, filepath, storage_service, key_service) diff --git a/warehouse/tuf/utils.py b/warehouse/tuf/utils.py index 8c6ae0c99abe..cc68c4199577 100644 --- a/warehouse/tuf/utils.py +++ b/warehouse/tuf/utils.py @@ -10,71 +10,156 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import os from contextlib import contextmanager from io import BytesIO -import tuf.formats -import tuf.repository_lib - from google.cloud.exceptions import GoogleCloudError, NotFound from securesystemslib.exceptions import StorageError +from securesystemslib.interface import generate_and_write_ed25519_keypair from securesystemslib.storage import FilesystemBackend, StorageBackendInterface -from tuf.api import metadata -from warehouse.tuf.constants import BIN_N_COUNT +from warehouse.config import Environment +from warehouse.tuf.constants import Role +from warehouse.tuf.repository import MetadataRepository, TargetsPayload + + +def set_expiration_for_role(config, role_name): + # If we're initializing TUF for development purposes, give + # every role a long expiration time so that developers don't have to + # continually re-initialize it. + if config.registry.settings["warehouse.env"] == Environment.development: + return datetime.datetime.now().replace(microsecond=0) + datetime.timedelta( + seconds=config.registry.settings["tuf.development_metadata_expiry"] + ) + else: + return datetime.datetime.now().replace(microsecond=0) + datetime.timedelta( + seconds=config.registry.settings[f"tuf.{role_name}.expiry"] + ) + + +def create_dev_keys(password, filepath) -> None: + generate_and_write_ed25519_keypair(password, filepath=filepath) def make_fileinfo(file, custom=None): """ - Given a warehouse.packaging.models.File, create a TUF-compliant - "fileinfo" dictionary suitable for addition to a delegated bin. + Create a TUF-compliant "fileinfo" dictionary suitable for addition to a + delegated bin. The optional "custom" kwarg can be used to supply additional custom metadata (e.g., metadata for indicating backsigning). """ - hashes = {"blake2b": file.blake2_256_digest} - fileinfo = tuf.formats.make_targets_fileinfo(file.size, hashes, custom=custom) + hashes = {"blake2b-256": file.blake2_256_digest} + fileinfo = dict() + fileinfo["length"] = file.size + fileinfo["hashes"] = hashes + if custom: + fileinfo["custom"] = custom return fileinfo -def bump_metadata(metadata, delta): - """ - Given a tuf.api.metadata.Signed, bump its version and expiration (with the given - timedelta). - """ - metadata.bump_version() - metadata.bump_expiration(delta=delta) +def repository_bump_snapshot(config, storage_service, key_service): + # Bumping the Snapshot role involves the following steps: + # 1. Initiate Metadata Repository. + # 2. Load the Snapshot Role. + # 3. Bump Snapshot role (and write to the Storage). + # 4. Bump Timestamp role using the new Snapshot Metadata. + # 1. Metadata Repository. + metadata_repository = MetadataRepository(storage_service, key_service) -def find_snapshot(timestamp, storage_backend): - """ - Given a tuf.api.metadata.Timestamp model, return the Metadata container - for the consistent snapshot that it references. - """ - snapshot_version = timestamp.meta["snapshot.json"]["version"] + # 2. Snapshot role metadata. + snapshot_metadata = metadata_repository.load_role(Role.SNAPSHOT.value) + + # 3. Bump Snapshot role metadata. + snapshot_metadata = metadata_repository.snapshot_bump_version( + snapshot_expires=set_expiration_for_role(config, Role.SNAPSHOT.value), + snapshot_metadata=snapshot_metadata, + store=True, + ) - return metadata.Metadata.from_json_file( - f"{snapshot_version}.snapshot.json", storage_backend + # 4. Bump Snapshot role etadata using Timestamp metadata. + metadata_repository.timestamp_bump_version( + snapshot_version=snapshot_metadata.signed.version, + timestamp_expires=set_expiration_for_role(config, Role.TIMESTAMP.value), + store=True, ) -def find_delegated_bin(filepath, snapshot, storage_backend): - """ - Given a new target filepath and a tuf.api.metadata.Snapshot model, - return a tuple of the bin name and tup.api.metadata.Metadata container for - the consistent delegated targets bin that the target belongs in. - """ +def repository_bump_bins_ns(config, storage_service, key_service): + # Bumping all of the delegated bin roles in the TUF repository involves + # the following steps: + # 1. Metadata Repository. + # 2. Load Snapshot role. + # 3. Load BIN-S role. + # 4. For each delegated hashed bin targets role in the BIN-S, fetch the + # role, bump, write back to the repo and update Snapshot role MetaFile. + # 5. Bump BIN-S and write back to repository. + # 6. Bump Snapshot and write back to repository. + # 7. Bump Timestamp role (using updated Snapshot) and write back to + # repository. + + # 1. Metadata Repository + metadata_repository = MetadataRepository(storage_service, key_service) + + # 2. Load Snapshot role. + snapshot_metadata = metadata_repository.load_role(Role.SNAPSHOT.value) + + # 3 Load BIN-S role. + bins_metadata = metadata_repository.load_role(Role.BINS.value) + + # 4. Fore each delegated hashed bin target role, bump and update Snapshot + for role in bins_metadata.signed.delegations.roles.keys(): + role_metadata = metadata_repository.load_role(role) + metadata_repository.bump_role_version( + rolename=role, + role_metadata=role_metadata, + role_expires=set_expiration_for_role(config, Role.BINS.value), + snapshot_metadata=snapshot_metadata, + key_rolename=Role.BIN_N.value, + store=True, + ) + snapshot_metadata = metadata_repository.snapshot_update_meta( + role, role_metadata.signed.version, snapshot_metadata + ) + + # 5. Bump BIN-S with updated metadata + snapshot_metadata = metadata_repository.bump_role_version( + rolename=Role.BINS.value, + role_metadata=bins_metadata, + role_expires=set_expiration_for_role(config, Role.BINS.value), + snapshot_metadata=snapshot_metadata, + key_rolename=None, + store=True, + ) + + # 6. Bump Snapshot with updated metadata + snapshot_metadata = metadata_repository.snapshot_bump_version( + snapshot_expires=set_expiration_for_role(config, Role.SNAPSHOT.value), + snapshot_metadata=snapshot_metadata, + store=True, + ) + + # Bump Timestamp with updated Snapshot metadata + metadata_repository.timestamp_bump_version( + snapshot_expires=snapshot_metadata.signed.version, + timestamp_expires=set_expiration_for_role(config, Role.TIMESTAMP.value), + store=True, + ) + - # TODO: This probably isn't using the right hash function. - filepath_hash = tuf.repository_lib.get_target_hash(filepath) - bin_name = tuf.repository_lib.find_bin_for_target_hash(filepath_hash, BIN_N_COUNT) - bin_version = snapshot.meta[f"{bin_name}.json"]["version"] +def repository_add_target(config, fileinfo, path, storage_service, key_service): + payload = [TargetsPayload(fileinfo, path)] - return bin_name, metadata.Metadata.from_json_file( - f"{bin_version}.{bin_name}.json", storage_backend + metadata_repository = MetadataRepository(storage_service, key_service) + metadata_repository.add_targets( + payload=payload, + timestamp_expires=set_expiration_for_role(config, Role.TIMESTAMP.value), + snapshot_expires=set_expiration_for_role(config, Role.SNAPSHOT.value), )