Skip to content

Commit

Permalink
Metadata 2.2 and 2.3 (#13606)
Browse files Browse the repository at this point in the history 
* Add `dynamic` to releases table for Metadata 2.2

Co-authored-by: Dustin Ingram <[email protected]>

* Add support for metadata 2.2

* Apply review suggestions

* Lint

* Update warehouse/forklift/legacy.py

Co-authored-by: Dustin Ingram <[email protected]>

* Move Supported-Platform to form

* Update warehouse/forklift/legacy.py

Co-authored-by: Dustin Ingram <[email protected]>

* Lint fixes

---------

Co-authored-by: Dustin Ingram <[email protected]>
Co-authored-by: Dominic Davis-Foster <[email protected]>
  • Loading branch information
@konstin @di @domdfcoding
3 people authored Feb 26, 2024
1 parent da6a528 commit 4532d55
Show file tree
Hide file tree
Showing 7 changed files with 321 additions and 6 deletions.
201 changes: 201 additions & 0 deletions tests/unit/forklift/test_legacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,72 @@ def test_validate_classifiers_invalid(self, db_request, data):
with pytest.raises(ValidationError):
legacy._validate_classifiers(form, field)

@pytest.mark.parametrize(
"data", [["Requires-Dist"], ["Requires-Dist", "Requires-Python"]]
)
def test_validate_dynamic_valid(self, db_request, data):
form = pretend.stub()
field = pretend.stub(data=data)

legacy._validate_dynamic(form, field)

@pytest.mark.parametrize(
"data",
[
["Version"],
["Name"],
["Version", "Name"],
["Provides-Extra", "I-Am-Not-Metadata"],
],
)
def test_validate_dynamic_invalid(self, db_request, data):
form = pretend.stub()
field = pretend.stub(data=data)

with pytest.raises(ValidationError):
legacy._validate_dynamic(form, field)

@pytest.mark.parametrize("data", [["dev"], ["dev-test"]])
def test_validate_provides_extras_valid(self, db_request, data):
form = pretend.stub(
provides_extra=pretend.stub(data=data),
metadata_version=pretend.stub(data="2.3"),
)
field = pretend.stub(data=data)

legacy._validate_provides_extras(form, field)

@pytest.mark.parametrize("data", [["dev_test"], ["dev.lint", "dev--test"]])
def test_validate_provides_extras_invalid(self, db_request, data):
form = pretend.stub(
provides_extra=pretend.stub(data=data),
metadata_version=pretend.stub(data="2.3"),
)
field = pretend.stub(data=data)

with pytest.raises(ValidationError):
legacy._validate_provides_extras(form, field)

@pytest.mark.parametrize("data", [["dev"], ["dev-test"]])
def test_validate_provides_extras_valid_2_2(self, db_request, data):
form = pretend.stub(
provides_extra=pretend.stub(data=data),
metadata_version=pretend.stub(data="2.2"),
)
field = pretend.stub(data=data)

legacy._validate_provides_extras(form, field)

@pytest.mark.parametrize("data", [["dev_test"], ["dev.lint", "dev--test"]])
def test_validate_provides_extras_invalid_2_2(self, db_request, data):
form = pretend.stub(
provides_extra=pretend.stub(data=data),
metadata_version=pretend.stub(data="2.2"),
)
field = pretend.stub(data=data)

legacy._validate_provides_extras(form, field)


def test_construct_dependencies():
types = {"requires": DependencyKind.requires, "provides": DependencyKind.provides}
Expand Down Expand Up @@ -478,6 +544,26 @@ def test_requires_python(self):
form = legacy.MetadataForm(MultiDict({"requires_python": ">= 3.5"}))
form.requires_python.validate(form)

@pytest.mark.parametrize(
"data",
[
{
"filetype": "bdist_wheel",
"metadata_version": "2.1",
"dynamic": "requires",
},
{
"metadata_version": "1.2",
"sha256_digest": "dummy",
"dynamic": "requires",
},
],
)
def test_dynamic_wrong_metadata_version(self, data):
form = legacy.MetadataForm(MultiDict(data))
with pytest.raises(ValidationError):
form.full_validate()


class TestFileValidation:
def test_defaults_to_true(self):
Expand Down Expand Up @@ -3417,6 +3503,121 @@ def test_upload_succeeds_creates_release(
),
]

@pytest.mark.parametrize(
"version, expected_version",
[
("1.0", "1.0"),
("v1.0", "1.0"),
],
)
def test_upload_succeeds_creates_release_metadata_2_3(
self, pyramid_config, db_request, metrics, version, expected_version
):
user = UserFactory.create()
EmailFactory.create(user=user)
project = ProjectFactory.create()
RoleFactory.create(user=user, project=project)

db_request.db.add(Classifier(classifier="Environment :: Other Environment"))
db_request.db.add(Classifier(classifier="Programming Language :: Python"))

filename = "{}-{}.tar.gz".format(project.name, "1.0")

pyramid_config.testing_securitypolicy(identity=user)
db_request.user = user
db_request.user_agent = "warehouse-tests/6.6.6"
db_request.POST = MultiDict(
{
"metadata_version": "2.3",
"name": project.name,
"version": version,
"summary": "This is my summary!",
"filetype": "sdist",
"md5_digest": _TAR_GZ_PKG_MD5,
"content": pretend.stub(
filename=filename,
file=io.BytesIO(_TAR_GZ_PKG_TESTDATA),
type="application/tar",
),
"supported_platform": "i386-win32-2791",
}
)
db_request.POST.extend(
[
("classifiers", "Environment :: Other Environment"),
("classifiers", "Programming Language :: Python"),
("requires_dist", "foo"),
("requires_dist", "bar (>1.0)"),
("project_urls", "Test, https://example.com/"),
("requires_external", "Cheese (>1.0)"),
("provides_extra", "testing"),
("provides_extra", "plugin"),
("dynamic", "Supported-Platform"),
]
)

storage_service = pretend.stub(store=lambda path, filepath, meta: None)
db_request.find_service = lambda svc, name=None, context=None: {
IFileStorage: storage_service,
IMetricsService: metrics,
}.get(svc)

resp = legacy.file_upload(db_request)

assert resp.status_code == 200

# Ensure that a Release object has been created.
release = (
db_request.db.query(Release)
.filter(
(Release.project == project) & (Release.version == expected_version)
)
.one()
)
assert release.summary == "This is my summary!"
assert release.classifiers == [
"Environment :: Other Environment",
"Programming Language :: Python",
]
assert set(release.requires_dist) == {"foo", "bar (>1.0)"}
assert release.project_urls == {"Test": "https://example.com/"}
assert set(release.requires_external) == {"Cheese (>1.0)"}
assert release.version == expected_version
assert release.canonical_version == "1"
assert release.uploaded_via == "warehouse-tests/6.6.6"
assert set(release.provides_extra) == {"testing", "plugin"}
assert set(release.dynamic) == {"Supported-Platform"}

# Ensure that a File object has been created.
db_request.db.query(File).filter(
(File.release == release) & (File.filename == filename)
).one()

# Ensure that a Filename object has been created.
db_request.db.query(Filename).filter(Filename.filename == filename).one()

# Ensure that all of our journal entries have been created
journals = (
db_request.db.query(JournalEntry)
.options(joinedload(JournalEntry.submitted_by))
.order_by("submitted_date", "id")
.all()
)
assert [(j.name, j.version, j.action, j.submitted_by) for j in journals] == [
(
release.project.name,
release.version,
"new release",
user,
),
(
release.project.name,
release.version,
f"add source file {filename}",
user,
),
]

def test_all_valid_classifiers_can_be_created(self, db_request):
for classifier in classifiers:
db_request.db.add(Classifier(classifier=classifier))
Expand Down
8 changes: 8 additions & 0 deletions tests/unit/legacy/api/test_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ def test_renders(self, pyramid_config, db_request, db_session):
"docs_url": "/the/fake/url/",
"download_url": None,
"downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
"dynamic": None,
"home_page": None,
"keywords": None,
"license": None,
Expand All @@ -242,6 +243,7 @@ def test_renders(self, pyramid_config, db_request, db_session):
"platform": None,
"project_url": "/the/fake/url/",
"project_urls": expected_urls,
"provides_extra": None,
"release_url": "/the/fake/url/",
"requires_dist": None,
"requires_python": None,
Expand Down Expand Up @@ -483,6 +485,8 @@ def test_detail_renders(self, pyramid_config, db_request, db_session):
description=DescriptionFactory.create(
content_type=description_content_type
),
dynamic=["Platform", "Supported-Platform"],
provides_extra=["testing", "plugin"],
)
]

Expand Down Expand Up @@ -540,6 +544,7 @@ def test_detail_renders(self, pyramid_config, db_request, db_session):
"docs_url": "/the/fake/url/",
"download_url": None,
"downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
"dynamic": ["Platform", "Supported-Platform"],
"home_page": None,
"keywords": None,
"license": None,
Expand All @@ -550,6 +555,7 @@ def test_detail_renders(self, pyramid_config, db_request, db_session):
"platform": None,
"project_url": "/the/fake/url/",
"project_urls": expected_urls,
"provides_extra": ["testing", "plugin"],
"release_url": "/the/fake/url/",
"requires_dist": None,
"requires_python": None,
Expand Down Expand Up @@ -630,6 +636,7 @@ def test_minimal_renders(self, pyramid_config, db_request):
"docs_url": None,
"download_url": None,
"downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
"dynamic": None,
"home_page": None,
"keywords": None,
"license": None,
Expand All @@ -640,6 +647,7 @@ def test_minimal_renders(self, pyramid_config, db_request):
"platform": None,
"project_url": "/the/fake/url/",
"project_urls": None,
"provides_extra": None,
"release_url": "/the/fake/url/",
"requires_dist": None,
"requires_python": None,
Expand Down
56 changes: 55 additions & 1 deletion warehouse/forklift/legacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
Dependency,
DependencyKind,
Description,
DynamicFieldsEnum,
File,
Filename,
JournalEntry,
Expand Down Expand Up @@ -395,6 +396,37 @@ def _validate_classifiers(form, field):
)


def _validate_dynamic(_form, field):
declared_dynamic_fields = {str.title(k) for k in field.data or []}
disallowed_dynamic_fields = {"Name", "Version", "Metadata-Version"}
if invalid := (declared_dynamic_fields & disallowed_dynamic_fields):
raise wtforms.validators.ValidationError(
f"The following metadata field(s) are valid, "
f"but cannot be marked as dynamic: {invalid!r}",
)
allowed_dynamic_fields = set(DynamicFieldsEnum.enums)
if invalid := (declared_dynamic_fields - allowed_dynamic_fields):
raise wtforms.validators.ValidationError(
f"The following metadata field(s) are not valid "
f"and cannot be marked as dynamic: {invalid!r}"
)


_extra_name_re = re.compile("^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$")


def _validate_provides_extras(form, field):
metadata_version = packaging.version.Version(form.metadata_version.data)

if metadata_version >= packaging.version.Version("2.3"):
if invalid := [
name for name in field.data or [] if not _extra_name_re.match(name)
]:
raise wtforms.validators.ValidationError(
f"The following Provides-Extra value(s) are invalid: {invalid!r}"
)


def _construct_dependencies(form, types):
for name, kind in types.items():
for item in getattr(form, name).data:
Expand All @@ -419,7 +451,7 @@ class MetadataForm(forms.Form):
# Note: This isn't really Metadata 2.0, however bdist_wheel
# claims it is producing a Metadata 2.0 metadata when in
# reality it's more like 1.2 with some extensions.
["1.0", "1.1", "1.2", "2.0", "2.1"],
["1.0", "1.1", "1.2", "2.0", "2.1", "2.2", "2.3"],
message="Use a known metadata version.",
),
],
Expand Down Expand Up @@ -470,6 +502,9 @@ class MetadataForm(forms.Form):
author = wtforms.StringField(
description="Author", validators=[wtforms.validators.Optional()]
)
supported_platform = wtforms.StringField(
description="Supported-Platform", validators=[wtforms.validators.Optional()]
)
description_content_type = wtforms.StringField(
description="Description-Content-Type",
validators=[wtforms.validators.Optional(), _validate_description_content_type],
Expand All @@ -495,6 +530,10 @@ class MetadataForm(forms.Form):
description="Classifier",
validators=[_validate_no_deprecated_classifiers, _validate_classifiers],
)
dynamic = ListField(
description="Dynamic",
validators=[_validate_dynamic],
)
platform = wtforms.StringField(
description="Platform", validators=[wtforms.validators.Optional()]
)
Expand Down Expand Up @@ -564,6 +603,10 @@ class MetadataForm(forms.Form):
description="Requires-Dist",
validators=[wtforms.validators.Optional(), _validate_legacy_dist_req_list],
)
provides_extra = ListField(
description="Provides-Extra",
validators=[wtforms.validators.Optional(), _validate_provides_extras],
)
provides_dist = ListField(
description="Provides-Dist",
validators=[wtforms.validators.Optional(), _validate_legacy_dist_req_list],
Expand Down Expand Up @@ -613,6 +656,15 @@ def full_validate(self):
"Include at least one message digest."
)

# Dynamic is only allowed with metadata version 2.2+
if self.dynamic.data:
metadata_version = packaging.version.Version(self.metadata_version.data)
if metadata_version and metadata_version < packaging.version.Version("2.2"):
raise wtforms.validators.ValidationError(
"'Dynamic' is only allowed in metadata version 2.2 and higher, "
f"but you declared {self.metadata_version.data}"
)


def _validate_filename(filename, filetype):
# Our object storage does not tolerate some specific characters
Expand Down Expand Up @@ -1107,6 +1159,8 @@ def file_upload(request):
"home_page",
"download_url",
"requires_python",
"dynamic",
"provides_extra",
}
},
uploader=request.user if request.user else None,
Expand Down
Loading

0 comments on commit 4532d55

Please sign in to comment.