From 8703dd9e03a7077b847d71c9f93ef41af7dd7c9a Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Thu, 15 Dec 2022 14:02:52 +0100 Subject: [PATCH 1/4] PyPI/Warehouse using RSTUF Adds the RSTUF in the Warehouse infrastructure * Include the RSTUF Ceremony payload file - It is generated using `rstuf admin ceremony`, and the keys * Add the development dependencies - RSTUF CLI and dependencies * Include RSTUF components to the `docker-compose.yml` - RSTUF uses the same Redis Server but uses unique Redis DB ids `1` and `2` - RSTUF uses the same PostgreSQL, but a specific database rstuf * Add the RSTUF environment configuration for development * Define the Makefile commands for RSTUF - `make tufinit` to bootstrap the RSTUF service - `make tufimport` to import all project packages to the RSTUF service * Define the basic commands for RSTUF within Warehouse - Command to import all existent packages and indexes to TUF metadata (`warehouse tuf dev import-all`) * Add TUF development documentation Signed-off-by: Kairo de Araujo --- .vscode/settings.json | 3 + Makefile | 10 +- dev/environment | 6 + dev/rstuf-bootstrap-payload.json | 90 +++++++++++ dev/tufkeys/online/online | 1 + dev/tufkeys/online/online.pub | 1 + dev/tufkeys/root/root1 | 1 + dev/tufkeys/root/root1.pub | 1 + dev/tufkeys/root/root2 | 1 + dev/tufkeys/root/root2.pub | 1 + docker-compose.yml | 67 +++++++++ docs/dev/development/getting-started.rst | 21 +++ requirements/dev.txt | 5 + warehouse/cli/tuf.py | 181 +++++++++++++++++++++++ warehouse/config.py | 9 ++ warehouse/packaging/tasks.py | 7 + warehouse/packaging/utils.py | 3 +- warehouse/tuf/__init__.py | 17 +++ 18 files changed, 423 insertions(+), 2 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 dev/rstuf-bootstrap-payload.json create mode 100644 dev/tufkeys/online/online create mode 100755 dev/tufkeys/online/online.pub create mode 100644 dev/tufkeys/root/root1 create mode 100755 dev/tufkeys/root/root1.pub create mode 100644 dev/tufkeys/root/root2 create mode 100755 dev/tufkeys/root/root2.pub create mode 100644 warehouse/cli/tuf.py create mode 100644 warehouse/tuf/__init__.py diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000000..a7d0fc7b77f6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "esbonio.sphinx.confDir": "" +} \ No newline at end of file diff --git a/Makefile b/Makefile index b6231b5d7f05..dfa6e7de5c77 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,14 @@ reindex: .state/docker-build-base shell: .state/docker-build-base docker compose run --rm web python -m warehouse shell +tufinit: + docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'" + docker compose restart rstuf-worker01 rstuf-worker02 + docker compose run --rm web rstuf admin ceremony -b -u -f /opt/warehouse/src/dev/rstuf-bootstrap-payload.json --upload-server http://rstuf-api + +tufimport: + docker-compose run --rm web python -m warehouse tuf dev import-all + dbshell: .state/docker-build-base docker compose run --rm web psql -h db -d warehouse -U postgres @@ -131,4 +139,4 @@ purge: stop clean stop: docker compose stop -.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations +.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations tufinit tufimport diff --git a/dev/environment b/dev/environment index 0f22c75d5cbc..6eeb074bdc68 100644 --- a/dev/environment +++ b/dev/environment @@ -70,3 +70,9 @@ OIDC_AUDIENCE=pypi # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI RECAPTCHA_SECRET_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe + +TUF_METADATA_URL="http://files:9001/metadata/" +TUF_API_URL="http://rstuf-api/api/v1/" +TUF_DATABASE_URL="postgresql://postgres@db/rstuf" +TUF_ROOT_SECRET="an insecure private key password" +TUF_ONLINE_SECRET="an insecure private key password" diff --git a/dev/rstuf-bootstrap-payload.json b/dev/rstuf-bootstrap-payload.json new file mode 100644 index 000000000000..f53809fa6bc7 --- /dev/null +++ b/dev/rstuf-bootstrap-payload.json @@ -0,0 +1,90 @@ +{ + "settings": { + "expiration": { + "root": 365, + "targets": 365, + "snapshot": 1, + "timestamp": 1, + "bins": 1 + }, + "services": { + "number_of_delegated_bins": 256, + "targets_base_url": "http://127.0.0.1:9001/simple/", + "targets_online_key": true + } + }, + "metadata": { + "root": { + "signatures": [ + { + "keyid": "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d", + "sig": "6fe3f661a40677df1ff5fac724cf3a47c826224be5ff9e1099cb76f826bac64722fa5e8120ad7eb032565a75a561d69255985b9de4ec25bb115710e8d3602d0b" + }, + { + "keyid": "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e", + "sig": "12485c76a748feed1ffdef59c24ba3258e56a20304207ae42138fff2c8c7314a14fb8f0beb7adfe85e78aebfc75200bac233a18a02d8c79ff06813f3900ff50e" + } + ], + "signed": { + "_type": "root", + "version": 1, + "spec_version": "1.0.30", + "expires": "2024-06-11T16:40:02Z", + "consistent_snapshot": true, + "keys": { + "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "ac5cd92ec491fea3f0b4c8a04af3fb957b5fc8965a79379131cfa4581905739f" + }, + "name": "root key 1" + }, + "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "6d112f8658d1d8f42b17a263641bf7bd8940c97f25f9ea83d3aa609ec5fe9a91" + }, + "name": "root key 2" + }, + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "41df147e582fb6c14445da4db011b7d7d03824ea7b64aef5bb3aa8a57269b327" + }, + "name": "online key v1" + } + }, + "roles": { + "root": { + "keyids": [ + "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d", + "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e" + ], + "threshold": 1 + }, + "timestamp": { + "keyids": [ + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2" + ], + "threshold": 1 + }, + "snapshot": { + "keyids": [ + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2" + ], + "threshold": 1 + }, + "targets": { + "keyids": [ + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2" + ], + "threshold": 1 + } + } + } + } + } +} \ No newline at end of file diff --git a/dev/tufkeys/online/online b/dev/tufkeys/online/online new file mode 100644 index 000000000000..e77865087230 --- /dev/null +++ b/dev/tufkeys/online/online @@ -0,0 +1 @@ +efd5e924987f59b3700a4188b83ae4dd@@@@100000@@@@c3cf5853b7cd2250cb72a1c0b4141c7367acddd3bcb6e96f6ec560f5f50e1c9e@@@@2480ed147201dd75b2d8f20b4a56534c@@@@58e914497e885078dbd8b5e6d10c1c435d8b1f83b53c35c8aa27fd5c703bcd3cc4b7ec8af0ef1d8444f5cad2a54093831944ac425777133f91c98df71018932f3fae77533f8f489f2bdbc63c0faddf2a00a63da37bd292f3ce7b35e86b7ddd90d0f2d92eaa9a264fda9eeb85f714b6745a9ff5ea3e3cd466d94557b0dce8fde3503a12ffd4ddfa0beaf5e509ce3514d071dc26af385dcd23f239711efbc86b2f736027f7940f9a3a786cb3a329158e5a0487ce50ee3a5ed4a032d6e556181a9ffb26c20800d0b4cd0b2149afee333986c0722101603f1d144d4df14a493376d516e7cbd54aa070f96c630672e8b2e8c89c7fe6d44d6cae87188289bdc8361635c613a06cd81f4b6f630938f003aa224c1ff3b31ec01c78fe5e85fcefcfe2d3beb33213b5e244c21b73841d46f47b9360e7bd6c75b8af6aa398dffdae5bce50795d57d37a3df0b3088392c1d050a46302 \ No newline at end of file diff --git a/dev/tufkeys/online/online.pub b/dev/tufkeys/online/online.pub new file mode 100755 index 000000000000..4c6ee59bb1c5 --- /dev/null +++ b/dev/tufkeys/online/online.pub @@ -0,0 +1 @@ +{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "41df147e582fb6c14445da4db011b7d7d03824ea7b64aef5bb3aa8a57269b327"}} \ No newline at end of file diff --git a/dev/tufkeys/root/root1 b/dev/tufkeys/root/root1 new file mode 100644 index 000000000000..430db3754b70 --- /dev/null +++ b/dev/tufkeys/root/root1 @@ -0,0 +1 @@ +0b81cc88ce39650626eba6a9b0420dad@@@@100000@@@@875fed5a914d59843f5280260145405c7599026e915efca75c7773b49b33b2e6@@@@bfb692965db58b72b5f1e9bceaeccc37@@@@81d62e800c1a22b8a82b7c795a2b77990494577dc098755b78d02eac175d3b96c5a5d4cc6e08c17410069118840c32443b7be43c3663963bfdf58f813cdcfd33a29770392ad35c05df7edbd51aaa0e6a6122b752cc876628633d10078fcf494818cf62abe41591ce9543abc4d0ea7a8e731097f1b97e2915358c015d61dadbdefe9a9be8138bdd87d89b952e89817951b60fee1089dcdadd4937e638571f47bd1258276803163797174dee1e963c731f40affaf35804117a5ee555767ea05bb165a3d4751a40d6e0dfc519049e2346dfcd09f851637328883251027ef3bb1566a1c4b1b95f127a636d0499a28fa52eade9e53d9560879ed56346582045750403978f575a4d753abd6e00e46c264f073e858693210ae04df8ce0520ca6fac64a668f61000e9a4b16c29cbcaf96dd35d5489bf1e36cbef3ee2927dffaf3ba822532653542a4516c3cacb3db4e2bca825cf \ No newline at end of file diff --git a/dev/tufkeys/root/root1.pub b/dev/tufkeys/root/root1.pub new file mode 100755 index 000000000000..d92c86c56945 --- /dev/null +++ b/dev/tufkeys/root/root1.pub @@ -0,0 +1 @@ +{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "ac5cd92ec491fea3f0b4c8a04af3fb957b5fc8965a79379131cfa4581905739f"}} \ No newline at end of file diff --git a/dev/tufkeys/root/root2 b/dev/tufkeys/root/root2 new file mode 100644 index 000000000000..7cac09b699c8 --- /dev/null +++ b/dev/tufkeys/root/root2 @@ -0,0 +1 @@ +370fd8314f62bf0743f150e9d9ec1883@@@@100000@@@@b89bb6032fa26cba87e51649a6e83df8a6ff92b81e2616a2af2f3794ce96aee2@@@@67c0d2cde784d5d354817ef42321d75d@@@@2558be0a0ed1b3db89be41669bb6657dcb85e05b1e41a2d57319fb715779cc230940ea88614cf96bf0f0f07a1f8726a780bf2003013c28c36956f597238d64502c3d063e8cf0d953f883f41f7787bcc7c233ca9e6c08e0fc0fdb988f99de80e456dc80d86f087a7535d5e6bed7db11feb4af247c04e01c3b7c0e658cfd6fb170e6370240cc7b0f9cfe0d15723122ae70c56d10487cc19b4dacb047ac8194cc1435a2e687bdcf20f2b4aca227581b3939b0c8aa712d8237bc8dbd977d64c4548359c75d0dad452e0a5517ce02da0db1bf5a077782e7997126f789e3ac93e3cf57feb08b9bf988ba8ff8b42dbf09bc5bbafbd8366d3ab1ebc0fe03899a48abdc55f324d96ef3a70265a728d5ce06f9d35c7b93ad399902be82f87f82f1eecbb1d4666031d515a2f9f14d6160560a9b505a444af8a79358e46609cfa339df8adacca9ee7cf7643b71da535032ae79934e7d \ No newline at end of file diff --git a/dev/tufkeys/root/root2.pub b/dev/tufkeys/root/root2.pub new file mode 100755 index 000000000000..4aa673c5302b --- /dev/null +++ b/dev/tufkeys/root/root2.pub @@ -0,0 +1 @@ +{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "6d112f8658d1d8f42b17a263641bf7bd8940c97f25f9ea83d3aa609ec5fe9a91"}} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a1a813d8b4aa..e6f3a1ecf1e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ volumes: policies: vault: caches: + rstuf-worker-data: + rstuf-metadata: services: vault: @@ -110,6 +112,8 @@ services: # Included to support linters during development - ./gunicorn-prod.conf.py:/opt/warehouse/src/gunicorn-prod.conf.py:z - ./gunicorn-uploads.conf.py:/opt/warehouse/src/gunicorn-uploads.conf.py:z + - rstuf-metadata:/var/opt/warehouse/metadata + - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z web: image: warehouse:docker-compose @@ -139,6 +143,7 @@ services: - packages-archive:/var/opt/warehouse/packages-archive - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - rstuf-metadata:/var/opt/warehouse/metadata ports: - "9001:9001" @@ -157,6 +162,68 @@ services: ARCHIVE_FILES_BACKEND: "warehouse.packaging.services.LocalArchiveFileStorage path=/var/opt/warehouse/packages-archive/ url=http://files:9001/packages-archive/{path}" SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}" + rstuf-worker01: + image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest + volumes: + - rstuf-worker-data:/data + - ./dev/rstuf-workers-supervisor.conf:/opt/repository-service-tuf/supervisor.conf:z + - rstuf-metadata:/var/opt/repository-service-tuf/storage + - ./dev/tufkeys/online:/var/opt/repository-service-tuf/keystorage + environment: + - RSTUF_STORAGE_BACKEND=LocalStorage + - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage + - RSTUF_KEYVAULT_BACKEND=LocalKeyVault + - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage + - RSTUF_LOCAL_KEYVAULT_KEYS=online,an insecure private key password + - RSTUF_BROKER_SERVER=redis://redis/1 + - RSTUF_REDIS_SERVER=redis://redis + - RSTUF_REDIS_SERVER_DB_RESULT=1 + - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf + healthcheck: + test: "exit 0" + restart: always + tty: true + depends_on: + db: + condition: service_healthy + + rstuf-worker02: + image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest + volumes: + - rstuf-worker-data:/data + - ./dev/rstuf-workers-supervisor.conf:/opt/repository-service-tuf/supervisor.conf:z + - rstuf-metadata:/var/opt/repository-service-tuf/storage + - ./dev/tufkeys/online:/var/opt/repository-service-tuf/keystorage + environment: + - RSTUF_STORAGE_BACKEND=LocalStorage + - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage + - RSTUF_KEYVAULT_BACKEND=LocalKeyVault + - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage + - RSTUF_LOCAL_KEYVAULT_KEYS=online,an insecure private key password + - RSTUF_BROKER_SERVER=redis://redis/1 + - RSTUF_REDIS_SERVER=redis://redis + - RSTUF_REDIS_SERVER_DB_RESULT=1 + - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf + healthcheck: + test: "exit 0" + restart: always + tty: true + depends_on: + db: + condition: service_healthy + + rstuf-api: + image: ghcr.io/repository-service-tuf/repository-service-tuf-api:latest + ports: + - 8001:80 + environment: + - RSTUF_BROKER_SERVER=redis://redis/1 + - RSTUF_REDIS_SERVER=redis://redis + - RSTUF_REDIS_SERVER_DB_RESULT=1 + - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + static: build: context: . diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index 2d2e9d9bbb48..b740db350aa4 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -249,6 +249,27 @@ or that the ``static`` container has finished compiling the static assets: or maybe something else. +Running the TUF Initialization (PEP 458) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console + + make tufinit + +This command will: + +* create a new Postgres RSTUF database +* restart the RSTUF services and run database migrations +* use a `RSTUF ceremony payload and bootstrap the TUF Repository + `_, + + +Optionally, you can import all examples of packages from the Warehouse +development database to the TUF Metadata + +.. code-block:: console + + make tufimport Viewing Warehouse in a browser ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/requirements/dev.txt b/requirements/dev.txt index c3ecdfbb0676..ff39594ae826 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,3 +2,8 @@ asyncudp>=0.7 hupper>=1.9 pip-tools>=1.0 pyramid_debugtoolbar>=2.5 +repository-service-tuf==0.3.0a1 +securesystemslib +dynaconf +rich-click +commonmark \ No newline at end of file diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py new file mode 100644 index 000000000000..e44d0ae516a7 --- /dev/null +++ b/warehouse/cli/tuf.py @@ -0,0 +1,181 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from datetime import datetime + +import click +import requests + +from sqlalchemy import MetaData, Table, create_engine +from sqlalchemy.dialects.sqlite import insert +from tuf.api.metadata import Delegations, Metadata + +from warehouse.cli import warehouse +from warehouse.packaging.tasks import ( + generate_projects_simple_detail as _generate_projects_simple_detail, +) +from warehouse.packaging.utils import render_simple_detail + + +@warehouse.group() # pragma: no-branch +def tuf(): + """ + Manage Warehouse's TUF state. + """ + + +@tuf.group() +def dev(): + """ + TUF Development purposes commands + """ + + +def _fetch_succinct_roles(config): + targets = Metadata.from_file("/var/opt/warehouse/metadata/1.targets.json") + if targets.signed.delegations is None: + raise click.ClickException("Failed to get Targets Delegations") + + targets_delegations = targets.signed.delegations + if targets_delegations.succinct_roles is None: + raise click.ClickException("Failed to get Targets succinct roles") + succinct_roles = targets_delegations.succinct_roles + + return succinct_roles + + +def _rstuf_target_files(path, size, blake2_256_digest, target_role_id): + return { + "path": path, + "info": { + "length": size, + "hashes": {"blake2b-256": blake2_256_digest}, + }, + "published": False, + "action": "ADD", + "last_update": datetime.now(), + "targets_role": target_role_id, + } + + +def _get_target_role_id(db, table, succinct_roles, path): + return db.execute( + table.select().where( + table.c.rolename == succinct_roles.get_role_for_target(path) + ) + ).one()[0] + + +@dev.command() +@click.pass_obj +def import_all(config): + """ + Collect every PyPI package and add as targets. + + Collect the "paths" for every PyPI package and project details index and add + as targets. These are packages already in existence, so we'll add all directly + to RSTUF database as a import process + """ + # NOTE: RSTUF CLI has an import feature but is not compatible to the Warehouse + # SQLAlchemy, as workaround we implemented similar feature here. + # https://repository-service-tuf.readthedocs.io/en/v1.0.0a1-draft/guide/general/usage.html#importing-existing-targets # noqa + + if config.registry.settings["warehouse.env"] != "development": + raise click.Click("Command allowed for development environment only") + + from warehouse.db import Session + from warehouse.packaging.models import File, Project + + succinct_roles = _fetch_succinct_roles(config) + db = Session(bind=config.registry["sqlalchemy.engine"]) + engine = create_engine(config.registry.settings["tuf.database.url"]) + db_metadata = MetaData() + db_client = engine.connect() + rstuf_target_files = Table("rstuf_target_files", db_metadata, autoload_with=engine) + rstuf_target_roles = Table("rstuf_target_roles", db_metadata, autoload_with=engine) + click.echo("Getting all Projects and generating package indexes") + rstuf_db_data_indexes = [] + all_projects = db.query(Project).all() + request = config.task(_generate_projects_simple_detail).get_request() + request.db = db + click.echo("Generating simple detail index and adding to RSTUF tables") + for project in all_projects: + try: + blake2_256_digest, path, size = render_simple_detail( + project, request, store=True + ) + except OSError: + continue + rstuf_db_data_indexes.append( + _rstuf_target_files( + path, + size, + blake2_256_digest, + _get_target_role_id( + db_client, rstuf_target_roles, succinct_roles, path + ), + ) + ) + + # FIXME: why do we have duplicated for simple detail indexes? + db_client.execute( + insert(rstuf_target_files) + .values(rstuf_db_data_indexes) + .on_conflict_do_nothing() + ) + db.commit() + click.echo("Getting all Packages") + files = db.query(File).all() + click.echo("Importing all packages to RSTUF tables") + rstuf_db_data_packages = [] + for file in files: + rstuf_db_data_packages.append( + _rstuf_target_files( + file.path, + file.size, + file.blake2_256_digest, + _get_target_role_id( + db_client, rstuf_target_roles, succinct_roles, file.path + ), + ) + ) + db_client.execute(rstuf_target_files.insert(), rstuf_db_data_packages) + db.commit() + + # Publish all packages and indexes + click.echo("Publishing all packages and indexes in TUF metadata") + publish_targets_url = config.registry.settings["tuf.api.publish.url"] + publish_all_targets = requests.post(publish_targets_url) + + # Monitoring publish all packages task + if publish_all_targets.status_code != 202: + raise click.ClickException(f"Error: {publish_all_targets.text}") + + task_id = publish_all_targets.json()["data"]["task_id"] + click.echo(f"Publishing packages in TUF metadata. Task id: {task_id}") + click.echo("Waiting packages and indexes to be published.", nl=False) + task_url = f"{config.registry.settings['tuf.api.task.url']}?task_id={task_id}" + while True: + task_status = requests.get(task_url) + if task_status.status_code != 200: + raise click.ClickException(f"Error: {task_status.text}") + state = task_status.json()["data"]["state"] + if state == "STARTED" or state == "PENDING": + click.echo(".", nl=False) + time.sleep(5) + elif state == "SUCCESS": + click.echo("\nAll packages and indexes published.") + break + else: + raise click.ClickException(f"Error: unexpected state {state}") diff --git a/warehouse/config.py b/warehouse/config.py index a9548430ad6a..009c2152418e 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -269,6 +269,12 @@ def configure(settings=None): coercer=int, default=100, ) + maybe_set(settings, "tuf.database.url", "TUF_DATABASE_URL") + maybe_set(settings, "tuf.metadata.url", "TUF_METADATA_URL") + maybe_set(settings, "tuf.api.url", "TUF_API_URL") + maybe_set(settings, "tuf.root1.secret", "TUF_ROOT_SECRET") + maybe_set(settings, "tuf.root2.secret", "TUF_ROOT_SECRET") + maybe_set(settings, "tuf.online.secret", "TUF_ROOT_SECRET") maybe_set_compound(settings, "billing", "backend", "BILLING_BACKEND") maybe_set_compound(settings, "files", "backend", "FILES_BACKEND") maybe_set_compound(settings, "archive_files", "backend", "ARCHIVE_FILES_BACKEND") @@ -657,6 +663,9 @@ def configure(settings=None): # Allow the packaging app to register any services it has. config.include(".packaging") + # Configure TUF (RSTUF) + config.include(".tuf") + # Configure redirection support config.include(".redirects") diff --git a/warehouse/packaging/tasks.py b/warehouse/packaging/tasks.py index 7f6618a2f413..d631c9bcab5f 100644 --- a/warehouse/packaging/tasks.py +++ b/warehouse/packaging/tasks.py @@ -28,6 +28,7 @@ from warehouse.metrics import IMetricsService from warehouse.packaging.interfaces import IFileStorage from warehouse.packaging.models import Description, File, Project, Release, Role +from warehouse.packaging.utils import render_simple_detail from warehouse.utils import readme from warehouse.utils.row_counter import RowCount @@ -557,3 +558,9 @@ def populate_data_using_schema(file): json_rows, table_name, job_config=LoadJobConfig(schema=table_schema) ).result() break + + +@tasks.task(ignore_result=True, acks_late=True) +def generate_projects_simple_detail(request, projects): + for project in projects: + render_simple_detail(project, request, store=True) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 496f6704419d..539b3e9a9985 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -96,6 +96,7 @@ def render_simple_detail(project, request, store=False): simple_detail_path = ( f"{project.normalized_name}/{content_hash}.{project.normalized_name}.html" ) + simple_detail_size = len(content.encode("utf-8")) if store: storage = request.find_service(ISimpleStorage) @@ -122,4 +123,4 @@ def render_simple_detail(project, request, store=False): }, ) - return (content_hash, simple_detail_path) + return (content_hash, simple_detail_path, simple_detail_size) diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py new file mode 100644 index 000000000000..45d74f3c5061 --- /dev/null +++ b/warehouse/tuf/__init__.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def includeme(config): + api_base_url = config.registry.settings["tuf.api.url"] + config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"}) + config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"}) From 5fe46cbb5803edcfa1c8a713e3dcc74ff25f52b7 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Fri, 9 Jun 2023 09:58:29 +0200 Subject: [PATCH 2/4] Manage packages/project and simple details to TUF * Adding packages After adding a package to the Warehouse database, it generates and stores the Simple Index with a request to the RSTUF backend to include the package and its simple index in TUF Metadata. * Removing package or Project Release On PyPI Management, when a user removes a file or a project release it also removes it from TUF metadata and updates the simple details index. Co-authored-by: Lukas Puehringer Signed-off-by: Kairo de Araujo simplify code in warehouse.tuf.targets Signed-off-by: Kairo de Araujo --- .vscode/settings.json | 3 - warehouse/forklift/legacy.py | 5 ++ warehouse/manage/views/__init__.py | 15 +++++ warehouse/packaging/interfaces.py | 5 ++ warehouse/packaging/services.py | 30 ++++++++++ warehouse/packaging/utils.py | 10 ++++ warehouse/tuf/__init__.py | 1 + warehouse/tuf/targets.py | 91 ++++++++++++++++++++++++++++++ 8 files changed, 157 insertions(+), 3 deletions(-) delete mode 100644 .vscode/settings.json create mode 100644 warehouse/tuf/targets.py diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index a7d0fc7b77f6..000000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "esbonio.sphinx.confDir": "" -} \ No newline at end of file diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index ac6ace5d1e97..eb3dd561aa02 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -68,6 +68,7 @@ ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from warehouse.rate_limiting.interfaces import RateLimiterException +from warehouse.tuf import targets from warehouse.utils import http, readme from warehouse.utils.project import PROJECT_NAME_RE, validate_project_name from warehouse.utils.security_policy import AuthenticationMethod @@ -1434,6 +1435,9 @@ def file_upload(request): file_data = file_ request.db.add(file_) + # Add the project simple detail and file to TUF Metadata + task = targets.add_file(request, project, file_) + file_.record_event( tag=EventTag.File.FileAdd, request=request, @@ -1449,6 +1453,7 @@ def file_upload(request): if request.oidc_publisher else None, "project_id": str(project.id), + "tuf": task["data"]["task_id"], }, ) diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index 341a3d0330c3..a91bd2321f9e 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -123,6 +123,7 @@ RoleInvitationStatus, ) from warehouse.rate_limiting import IRateLimiter +from warehouse.tuf import targets from warehouse.utils.http import is_safe_url from warehouse.utils.paginate import paginate_url_factory from warehouse.utils.project import confirm_project, destroy_docs, remove_project @@ -1845,17 +1846,24 @@ def delete_project_release(self): ) ) + # Delete the project release (simple detail and files) from TUF Metadata + tasks = targets.delete_release(self.request, self.release) + self.release.project.record_event( tag=EventTag.Project.ReleaseRemove, request=self.request, additional={ "submitted_by": self.request.user.username, "canonical_version": self.release.canonical_version, + "tuf": ", ".join([task["data"]["task_id"] for task in tasks]), }, ) self.request.db.delete(self.release) + # Generate new project simple detail and add to TUF Metadata + targets.add_file(self.request, self.release.project) + self.request.session.flash( self.request._(f"Deleted release {self.release.version!r}"), queue="success" ) @@ -1937,6 +1945,9 @@ def _error(message): ) ) + # Delete the file and project simple detail from TUF metadata + task = targets.delete_file(self.request, self.release.project, release_file) + release_file.record_event( tag=EventTag.File.FileRemove, request=self.request, @@ -1945,6 +1956,7 @@ def _error(message): "canonical_version": self.release.canonical_version, "filename": release_file.filename, "project_id": str(self.release.project.id), + "tuf": task["data"]["task_id"], }, ) @@ -1969,6 +1981,9 @@ def _error(message): self.request.db.delete(release_file) + # Generate new project simple detail and add to TUF Metadata + targets.add_file(self.request, self.release.project) + self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) diff --git a/warehouse/packaging/interfaces.py b/warehouse/packaging/interfaces.py index 448620d2842f..ead8d85d1b87 100644 --- a/warehouse/packaging/interfaces.py +++ b/warehouse/packaging/interfaces.py @@ -44,6 +44,11 @@ def get_checksum(path): Return the md5 digest of the file at a given path as a lowercase string. """ + def get_blake2bsum(path): + """ + Return the blake2b digest of the file at a given path as a lowercase string. + """ + def store(path, file_path, *, meta=None): """ Save the file located at file_path to the file storage at the location diff --git a/warehouse/packaging/services.py b/warehouse/packaging/services.py index 356129f82bd9..15c536066469 100644 --- a/warehouse/packaging/services.py +++ b/warehouse/packaging/services.py @@ -76,6 +76,13 @@ def get_checksum(self, path): open(os.path.join(self.base, path), "rb").read(), usedforsecurity=False ).hexdigest() + def get_blake2bsum(self, path): + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(open(os.path.join(self.base, path), "rb").read()) + content_hash = content_hasher.hexdigest().lower() + + return content_hash + def store(self, path, file_path, *, meta=None): destination = os.path.join(self.base, path) os.makedirs(os.path.dirname(destination), exist_ok=True) @@ -197,6 +204,9 @@ def create_service(cls, context, request): prefix = request.registry.settings.get("files.prefix") return cls(bucket, prefix=prefix) + def get_blake2bsum(self, path): + raise NotImplementedError + class GenericS3BlobStorage(GenericBlobStorage): def get(self, path): @@ -249,6 +259,9 @@ def create_service(cls, context, request): prefix = request.registry.settings.get("files.prefix") return cls(bucket, prefix=prefix) + def get_blake2bsum(self, path): + raise NotImplementedError + @implementer(IFileStorage) class S3ArchiveFileStorage(GenericS3BlobStorage): @@ -260,6 +273,9 @@ def create_service(cls, context, request): prefix = request.registry.settings.get("archive_files.prefix") return cls(bucket, prefix=prefix) + def get_blake2bsum(self, path): + raise NotImplementedError + @implementer(IDocsStorage) class S3DocsStorage: @@ -307,6 +323,20 @@ def get_metadata(self, path): def get_checksum(self, path): raise NotImplementedError + @google.api_core.retry.Retry( + predicate=google.api_core.retry.if_exception_type( + google.api_core.exceptions.ServiceUnavailable + ) + ) + def get_blake2bsum(self, path): + path = self._get_path(path) + blob = self.bucket.blob(path) + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(blob.download_as_string()) + content_hash = content_hasher.hexdigest().lower() + + return content_hash + @google.api_core.retry.Retry( predicate=google.api_core.retry.if_exception_type( google.api_core.exceptions.ServiceUnavailable diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 539b3e9a9985..4cd7e1f66a07 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -124,3 +124,13 @@ def render_simple_detail(project, request, store=False): ) return (content_hash, simple_detail_path, simple_detail_size) + + +def current_simple_details_path(request, project): + storage = request.find_service(ISimpleStorage) + current_hash = storage.get_blake2bsum(f"{project.normalized_name}/index.html") + simple_detail_path = ( + f"{project.normalized_name}/{current_hash}.{project.normalized_name}.html" + ) + + return simple_detail_path diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 45d74f3c5061..1002529c4cdb 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -14,4 +14,5 @@ def includeme(config): api_base_url = config.registry.settings["tuf.api.url"] config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"}) + config.add_settings({"tuf.api.targets.url": f"{api_base_url}targets/"}) config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"}) diff --git a/warehouse/tuf/targets.py b/warehouse/tuf/targets.py new file mode 100644 index 000000000000..cb62b9feffd2 --- /dev/null +++ b/warehouse/tuf/targets.py @@ -0,0 +1,91 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from pyramid.httpexceptions import HTTPBadGateway + +from warehouse.packaging.models import File +from warehouse.packaging.utils import current_simple_details_path, render_simple_detail + + +def _payload(targets): + """Helper to create payload for POST or DELETE targets request.""" + return { + "targets": targets, + "publish_targets": True, + } + + +def _payload_targets_part(path, size, digest): + """Helper to create payload part for POST targets request.""" + return { + "path": path, + "info": { + "length": size, + "hashes": {"blake2b-256": digest}, + }, + } + + +def _handle(response): + """Helper to handle http response for POST or DELETE targets request.""" + if response.status_code != 202: + raise HTTPBadGateway(f"Unexpected TUF Server response: {response.text}") + + return response.json() + + +def add_file(request, project, file=None): + """Call RSTUF to add file and new project simple index to TUF targets metadata. + + NOTE: If called without file, only adds new project simple index. This + can be used to re-add project simple index, after deleting a file. + """ + targets = [] + digest, path, size = render_simple_detail(project, request, store=True) + simple_index_part = _payload_targets_part(path, size, digest) + targets.append(simple_index_part) + if file: + file_part = _payload_targets_part(file.path, file.size, file.blake2_256_digest) + targets.append(file_part) + + response = requests.post( + request.registry.settings["tuf.api.targets.url"], json=_payload(targets) + ) + + return _handle(response) + + +def delete_file(request, project, file): + """Call RSTUF to remove file and project simple index from TUF targets metadata. + + NOTE: Simple index needs to be added separately. + """ + index_path = current_simple_details_path(request, project) + targets = [file.path, index_path] + + response = requests.delete( + request.registry.settings["tuf.api.targets.url"], json=_payload(targets) + ) + + return _handle(response) + + +def delete_release(request, release): + files = request.db.query(File).filter(File.release_id == release.id).all() + + tasks = [] + for file in files: + tasks.append(delete_file(request, release.project, file)) + + return tasks From 2f52784eeaa90890a354ab75b064e37db1289817 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Sat, 29 Jul 2023 12:20:06 +0200 Subject: [PATCH 3/4] Reduce the number of delegated hash-bin dev Reduce the number of delegated hash-bin roles for the development enviroment. Signed-off-by: Kairo de Araujo --- dev/rstuf-bootstrap-payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/rstuf-bootstrap-payload.json b/dev/rstuf-bootstrap-payload.json index f53809fa6bc7..4201d00ec917 100644 --- a/dev/rstuf-bootstrap-payload.json +++ b/dev/rstuf-bootstrap-payload.json @@ -8,7 +8,7 @@ "bins": 1 }, "services": { - "number_of_delegated_bins": 256, + "number_of_delegated_bins": 8, "targets_base_url": "http://127.0.0.1:9001/simple/", "targets_online_key": true } From b86baea9f148f60187f248e6df72d7e3db22cca0 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Sat, 29 Jul 2023 12:23:59 +0200 Subject: [PATCH 4/4] Rename the TUF_API_URL to RSTUF_API_URL Rename the environment variable setting `TUF_API_URL` to `RSTUF_API_URL` as this API is provided by Repository Service for TUF (RSTUF). Signed-off-by: Kairo de Araujo --- dev/environment | 2 +- warehouse/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/environment b/dev/environment index 6eeb074bdc68..7f483e13cbb6 100644 --- a/dev/environment +++ b/dev/environment @@ -72,7 +72,7 @@ RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI RECAPTCHA_SECRET_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe TUF_METADATA_URL="http://files:9001/metadata/" -TUF_API_URL="http://rstuf-api/api/v1/" +RSTUF_API_URL="http://rstuf-api/api/v1/" TUF_DATABASE_URL="postgresql://postgres@db/rstuf" TUF_ROOT_SECRET="an insecure private key password" TUF_ONLINE_SECRET="an insecure private key password" diff --git a/warehouse/config.py b/warehouse/config.py index 009c2152418e..f9c35c87e694 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -271,7 +271,7 @@ def configure(settings=None): ) maybe_set(settings, "tuf.database.url", "TUF_DATABASE_URL") maybe_set(settings, "tuf.metadata.url", "TUF_METADATA_URL") - maybe_set(settings, "tuf.api.url", "TUF_API_URL") + maybe_set(settings, "tuf.api.url", "RSTUF_API_URL") maybe_set(settings, "tuf.root1.secret", "TUF_ROOT_SECRET") maybe_set(settings, "tuf.root2.secret", "TUF_ROOT_SECRET") maybe_set(settings, "tuf.online.secret", "TUF_ROOT_SECRET")