From 231be6e4e898a56799d19033e20606aaf936915b Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Sat, 3 Nov 2018 11:40:25 -0500 Subject: [PATCH 1/4] Create fuzzystrmatch extension --- .../56e9e630c748_add_fuzzystrmatch.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 warehouse/migrations/versions/56e9e630c748_add_fuzzystrmatch.py diff --git a/warehouse/migrations/versions/56e9e630c748_add_fuzzystrmatch.py b/warehouse/migrations/versions/56e9e630c748_add_fuzzystrmatch.py new file mode 100644 index 000000000000..5382ff9f6fc7 --- /dev/null +++ b/warehouse/migrations/versions/56e9e630c748_add_fuzzystrmatch.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Add fuzzystrmatch + +Revision ID: 56e9e630c748 +Revises: e82c3a017d60 +Create Date: 2018-08-28 19:00:47.606523 +""" + +from alembic import op + + +revision = "56e9e630c748" +down_revision = "e82c3a017d60" + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS fuzzystrmatch") + + +def downgrade(): + pass From 20151e6adbb1740a5f6c7adc4b67374ebcbd3b5c Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Sat, 3 Nov 2018 11:42:17 -0500 Subject: [PATCH 2/4] Add a Squat to the DB on upload If projects exist with a similar name as the project being created, add a Squat entry to the DB linking the two projects. --- tests/unit/forklift/test_legacy.py | 45 ++++++++++++++++ warehouse/admin/squats.py | 35 +++++++++++++ warehouse/forklift/legacy.py | 24 ++++++++- .../versions/eeb23d9b4d00_add_squats_table.py | 51 +++++++++++++++++++ 4 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 warehouse/admin/squats.py create mode 100644 warehouse/migrations/versions/eeb23d9b4d00_add_squats_table.py diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index cc6e6eefa542..95d3dfc146db 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -29,6 +29,7 @@ from wtforms.form import Form from wtforms.validators import ValidationError +from warehouse.admin.squats import Squat from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy from warehouse.packaging.interfaces import IFileStorage @@ -2779,6 +2780,50 @@ def test_upload_succeeds_creates_project(self, pyramid_config, db_request): ), ] + def test_upload_succeeds_creates_squats(self, pyramid_config, db_request): + pyramid_config.testing_securitypolicy(userid=1) + + squattee = ProjectFactory(name="example") + user = UserFactory.create() + EmailFactory.create(user=user) + + filename = "{}-{}.tar.gz".format("exmaple", "1.0") + + db_request.user = user + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": "exmaple", + "version": "1.0", + "filetype": "sdist", + "md5_digest": "335c476dc930b959dda9ec82bd65ef19", + "content": pretend.stub( + filename=filename, + file=io.BytesIO(b"A fake file."), + type="application/tar", + ), + } + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None: storage_service + db_request.remote_addr = "10.10.10.10" + db_request.user_agent = "warehouse-tests/6.6.6" + + resp = legacy.file_upload(db_request) + + assert resp.status_code == 200 + + # Ensure that a Project object has been created. + squatter = db_request.db.query(Project).filter(Project.name == "exmaple").one() + + # Ensure that a Squat object has been created. + squat = db_request.db.query(Squat).one() + + assert squat.squattee == squattee + assert squat.squatter == squatter + assert squat.reviewed is False + @pytest.mark.parametrize( ("emails_verified", "expected_success"), [ diff --git a/warehouse/admin/squats.py b/warehouse/admin/squats.py new file mode 100644 index 000000000000..6df274a5d32f --- /dev/null +++ b/warehouse/admin/squats.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, Text +from sqlalchemy import orm, sql + +from warehouse import db + + +class Squat(db.ModelBase): + + __tablename__ = "warehouse_admin_squat" + + id = Column(Integer, primary_key=True, nullable=False) + created = Column( + DateTime(timezone=False), nullable=False, server_default=sql.func.now() + ) + squatter_name = Column( + Text, ForeignKey("packages.name", onupdate="CASCADE", ondelete="CASCADE") + ) + squattee_name = Column( + Text, ForeignKey("packages.name", onupdate="CASCADE", ondelete="CASCADE") + ) + squatter = orm.relationship("Project", foreign_keys=[squatter_name], lazy=False) + squattee = orm.relationship("Project", foreign_keys=[squattee_name], lazy=False) + reviewed = Column(Boolean, nullable=False, server_default=sql.false()) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 2411b62c1b06..ace40eb6d268 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -39,6 +39,7 @@ from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound from warehouse import forms +from warehouse.admin.squats import Squat from warehouse.classifiers.models import Classifier from warehouse.packaging.interfaces import IFileStorage from warehouse.packaging.models import ( @@ -863,10 +864,29 @@ def file_upload(request): ), ) from None - # The project doesn't exist in our database, so we'll add it along with - # a role setting the current user as the "Owner" of the project. + # The project doesn't exist in our database, so first we'll check for + # projects with a similar name + squattees = ( + request.db.query(Project) + .filter( + func.levenshtein( + Project.normalized_name, func.normalize_pep426_name(form.name.data) + ) + <= 2 + ) + .all() + ) + + # Next we'll create the project project = Project(name=form.name.data) request.db.add(project) + + # Now that the project exists, add any squats which it is the squatter for + for squattee in squattees: + request.db.add(Squat(squatter=project, squattee=squattee)) + + # Then we'll add a role setting the current user as the "Owner" of the + # project. request.db.add(Role(user=request.user, project=project, role_name="Owner")) # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this diff --git a/warehouse/migrations/versions/eeb23d9b4d00_add_squats_table.py b/warehouse/migrations/versions/eeb23d9b4d00_add_squats_table.py new file mode 100644 index 000000000000..0870b4240b35 --- /dev/null +++ b/warehouse/migrations/versions/eeb23d9b4d00_add_squats_table.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +add squats table + +Revision ID: eeb23d9b4d00 +Revises: 56e9e630c748 +Create Date: 2018-11-03 06:05:42.158355 +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "eeb23d9b4d00" +down_revision = "56e9e630c748" + + +def upgrade(): + op.create_table( + "warehouse_admin_squat", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False + ), + sa.Column("squatter_name", sa.Text(), nullable=True), + sa.Column("squattee_name", sa.Text(), nullable=True), + sa.Column( + "reviewed", sa.Boolean(), server_default=sa.text("false"), nullable=False + ), + sa.ForeignKeyConstraint( + ["squattee_name"], ["packages.name"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["squatter_name"], ["packages.name"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("warehouse_admin_squat") From f140a1669ad887b6d969bb841ee5c68da63ac714 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Sat, 3 Nov 2018 11:46:42 -0500 Subject: [PATCH 3/4] Display Squats in admin UI, & make them reviewable Display a list of squats in order that have not been reviewed, and make them "reviewable"/dismissable --- tests/unit/admin/test_routes.py | 2 + tests/unit/admin/views/test_squats.py | 47 +++++++++++ warehouse/admin/routes.py | 4 + warehouse/admin/templates/admin/base.html | 5 ++ .../admin/templates/admin/squats/index.html | 80 +++++++++++++++++++ warehouse/admin/views/squats.py | 50 ++++++++++++ 6 files changed, 188 insertions(+) create mode 100644 tests/unit/admin/views/test_squats.py create mode 100644 warehouse/admin/templates/admin/squats/index.html create mode 100644 warehouse/admin/views/squats.py diff --git a/tests/unit/admin/test_routes.py b/tests/unit/admin/test_routes.py index 3cdbab6326b6..578b39e318f5 100644 --- a/tests/unit/admin/test_routes.py +++ b/tests/unit/admin/test_routes.py @@ -111,4 +111,6 @@ def test_includeme(): ), pretend.call("admin.flags", "/admin/flags/", domain=warehouse), pretend.call("admin.flags.edit", "/admin/flags/edit/", domain=warehouse), + pretend.call("admin.squats", "/admin/squats/", domain=warehouse), + pretend.call("admin.squats.review", "/admin/squats/review/", domain=warehouse), ] diff --git a/tests/unit/admin/views/test_squats.py b/tests/unit/admin/views/test_squats.py new file mode 100644 index 000000000000..aceb9733980b --- /dev/null +++ b/tests/unit/admin/views/test_squats.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from warehouse.admin.squats import Squat +from warehouse.admin.views import squats as views + +from ....common.db.packaging import ProjectFactory + + +class TestGetSquats: + def test_get_squats(self, db_request): + project_a = ProjectFactory() + project_b = ProjectFactory() + project_c = ProjectFactory() + squat = Squat(squattee=project_a, squatter=project_b) + reviewed_squat = Squat(squattee=project_a, squatter=project_c, reviewed=True) + db_request.db.add(squat) + db_request.db.add(reviewed_squat) + + assert views.get_squats(db_request) == {"squats": [squat]} + + +class TestReviewSquat: + def test_review_squat(self, db_request): + squat = Squat(squattee=ProjectFactory(), squatter=ProjectFactory()) + db_request.db.add(squat) + + db_request.db.flush() + + db_request.POST = {"id": squat.id} + db_request.route_path = lambda *a: "/the/redirect" + db_request.flash = lambda *a: None + + views.review_squat(db_request) + + db_request.db.flush() + + assert squat.reviewed is True diff --git a/warehouse/admin/routes.py b/warehouse/admin/routes.py index 1d748f50f681..c2dfd5d9a286 100644 --- a/warehouse/admin/routes.py +++ b/warehouse/admin/routes.py @@ -116,3 +116,7 @@ def includeme(config): # Flags config.add_route("admin.flags", "/admin/flags/", domain=warehouse) config.add_route("admin.flags.edit", "/admin/flags/edit/", domain=warehouse) + + # Squats + config.add_route("admin.squats", "/admin/squats/", domain=warehouse) + config.add_route("admin.squats.review", "/admin/squats/review/", domain=warehouse) diff --git a/warehouse/admin/templates/admin/base.html b/warehouse/admin/templates/admin/base.html index 12171f49303f..410d70fe1e60 100644 --- a/warehouse/admin/templates/admin/base.html +++ b/warehouse/admin/templates/admin/base.html @@ -120,6 +120,11 @@ Flags +
  • + + Squats + +
    • diff --git a/warehouse/admin/templates/admin/squats/index.html b/warehouse/admin/templates/admin/squats/index.html new file mode 100644 index 000000000000..da76b6afed80 --- /dev/null +++ b/warehouse/admin/templates/admin/squats/index.html @@ -0,0 +1,80 @@ +{# + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-#} +{% extends "admin/base.html" %} + +{% import "admin/utils/pagination.html" as pagination %} + +{% block title %}Squats{% endblock %} + +{% block breadcrumb %} +
  • Squats
    • +{% endblock %} + +{% set csrf_token = request.session.get_csrf_token() %} + +{% block content %} +
    +
    +

    List Squats

    +
    +
    + + + + + + + + + {% for squat in squats %} + + + + + + + + {% else %} + + + + {% endfor %} +
    Potential SquatterSquatting onCreatedCreated by
    + + {{ squat.squatter.normalized_name }} + + + + {{ squat.squattee.normalized_name }} + + {{ squat.created }} + {% for user in squat.squatter.users %} + + {{ user.username }} + {{ "," if not loop.last }} + {% endfor %} + +
    + + + +
    +
    +
    + No squats! +
    +
    +
    +
    +{% endblock content %} diff --git a/warehouse/admin/views/squats.py b/warehouse/admin/views/squats.py new file mode 100644 index 000000000000..62b9c2d0258b --- /dev/null +++ b/warehouse/admin/views/squats.py @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pyramid.httpexceptions import HTTPSeeOther +from pyramid.view import view_config + +from warehouse.admin.squats import Squat + + +@view_config( + route_name="admin.squats", + renderer="admin/squats/index.html", + permission="admin", + uses_session=True, +) +def get_squats(request): + return { + "squats": ( + request.db.query(Squat) + .filter(Squat.reviewed.is_(False)) + .order_by(Squat.created.desc()) + .all() + ) + } + + +@view_config( + route_name="admin.squats.review", + permission="admin", + request_method="POST", + uses_session=True, + require_methods=False, + require_csrf=True, +) +def review_squat(request): + squat = request.db.query(Squat).get(request.POST["id"]) + squat.reviewed = True + + request.session.flash("Squat marked as reviewed") + + return HTTPSeeOther(request.route_path("admin.squats")) From bdee88a1211d6d8c75bb06f27c85e2317d165230 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Sat, 3 Nov 2018 12:24:56 -0500 Subject: [PATCH 4/4] Include possible squatters/squattees in admin UI On the project detail page, show a list of potential squatters and squattees for a given project. --- tests/unit/admin/views/test_projects.py | 24 ++++++++-- .../templates/admin/projects/detail.html | 47 +++++++++++++++++++ warehouse/admin/views/projects.py | 18 ++++++- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/tests/unit/admin/views/test_projects.py b/tests/unit/admin/views/test_projects.py index 4676b5d2be94..d3617f685a59 100644 --- a/tests/unit/admin/views/test_projects.py +++ b/tests/unit/admin/views/test_projects.py @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime + import pretend import pytest import uuid @@ -88,12 +90,28 @@ def test_gets_project(self, db_request): [RoleFactory(project=project) for _ in range(5)], key=lambda x: (x.role_name, x.user.username), ) + delta = datetime.timedelta(days=1) + squatter = ProjectFactory( + name=project.name[:-1], created=project.created + delta + ) + squattee = ProjectFactory( + name=project.name[1:], created=project.created - delta + ) + db_request.db.add(squatter) + db_request.db.add(squattee) db_request.matchdict["project_name"] = str(project.normalized_name) result = views.project_detail(project, db_request) - assert result["project"] == project - assert result["maintainers"] == roles - assert result["journal"] == journals[:30] + assert result == { + "project": project, + "releases": [], + "maintainers": roles, + "journal": journals[:30], + "squatters": [squatter], + "squattees": [squattee], + "ONE_MB": views.ONE_MB, + "MAX_FILESIZE": views.MAX_FILESIZE, + } def test_non_normalized_name(self, db_request): project = ProjectFactory.create() diff --git a/warehouse/admin/templates/admin/projects/detail.html b/warehouse/admin/templates/admin/projects/detail.html index 79ad83cb048a..cb765714b6aa 100644 --- a/warehouse/admin/templates/admin/projects/detail.html +++ b/warehouse/admin/templates/admin/projects/detail.html @@ -46,6 +46,7 @@

    {{ project.name }}

    +

    Attributes:

    @@ -64,6 +65,7 @@

    {{ project.name }}

    +

    Maintainers:

    @@ -135,6 +137,7 @@
    +

    Releases:

    @@ -152,6 +155,7 @@
    Release version
    All releases
    +

    Journals:

    @@ -175,6 +179,49 @@
    All journals
    + {% if squatters %} +

    Projects that might be squatting on this project:

    +
    + + + + + + {% for project in squatters %} + + + + + {% endfor %} +
    namecreated
    + + {{ project.normalized_name }} + + {{ project.created }}
    +
    + {% endif %} + + {% if squattees %} +

    Projects that this project might be squatting on:

    +
    + + + + + + {% for project in squattees %} + + + + + {% endfor %} +
    namecreated
    + + {{ project.normalized_name }} + + {{ project.created }}
    +
    + {% endif %}
    diff --git a/warehouse/admin/views/projects.py b/warehouse/admin/views/projects.py index 1b566d388c4b..9bbd3c482aa0 100644 --- a/warehouse/admin/views/projects.py +++ b/warehouse/admin/views/projects.py @@ -15,7 +15,7 @@ from paginate_sqlalchemy import SqlalchemyOrmPage as SQLAlchemyORMPage from pyramid.httpexceptions import HTTPBadRequest, HTTPMovedPermanently, HTTPSeeOther from pyramid.view import view_config -from sqlalchemy import or_ +from sqlalchemy import or_, func from sqlalchemy.orm.exc import NoResultFound from warehouse.accounts.models import User @@ -107,11 +107,27 @@ def project_detail(project, request): ) ] + squattees = ( + request.db.query(Project) + .filter(Project.created < project.created) + .filter(func.levenshtein(Project.normalized_name, project.normalized_name) <= 2) + .all() + ) + + squatters = ( + request.db.query(Project) + .filter(Project.created > project.created) + .filter(func.levenshtein(Project.normalized_name, project.normalized_name) <= 2) + .all() + ) + return { "project": project, "releases": releases, "maintainers": maintainers, "journal": journal, + "squatters": squatters, + "squattees": squattees, "ONE_MB": ONE_MB, "MAX_FILESIZE": MAX_FILESIZE, }