Make changes necessary to deal with new term

README.md

@@ -46,16 +46,16 @@ docker compose exec backend bash
 While we use a background worker to run data pipelines on a regular schedule in production, you can also run them manually using the CLI:
 
 ```
-htv pipeline members --term=9
-htv pipeline sessions --term=9
-htv pipeline rcv-list --term=9 --date=2024-04-10
-htv pipeline press --date=2024-04-10
+htv pipeline members --term=10
+htv pipeline sessions --term=10
+htv pipeline rcv-list --term=10 --date=2024-07-16
+htv pipeline press --date=2024-07-16
 ```
 
 In order to run all data pipelines for an entire term, run the following command. Please note that this command may take multiple hours to complete.
 
 ```
-htv pipeline all --term=9
+htv pipeline all --term=10
 ```
 
 ---

backend/howtheyvote/analysis/votes.py

@@ -194,12 +194,17 @@ def empty_titles(self) -> Iterator[Fragment]:
 
     def member_votes_count(self) -> Iterator[Fragment]:
         brexit_date = datetime.date(2020, 2, 1)
+        tenth_term_date = datetime.date(2024, 7, 16)
         date = self.vote.timestamp.date()
         count = len(self.vote.member_votes)
 
         issues = []
 
-        if (date < brexit_date and count != 751) or (date >= brexit_date and count != 705):
+        if (
+            (date < brexit_date and count != 751)
+            or (date >= brexit_date and count != 705)
+            or (date >= tenth_term_date and count != 720)
+        ):
             issues = [DataIssue.MEMBER_VOTES_COUNT_MISMATCH]
 
         yield Fragment(

backend/howtheyvote/api/query.py

@@ -225,7 +225,7 @@ def handle(self) -> QueryResponse[T]:
 
         # Based on the IDs fetched from the search index, fetch full records
         # from the database
-        ids = [hit["id"] for hit in res["hits"]]
+        ids = [int(hit["id"]) for hit in res["hits"]]
 
         # Remove the extra item fetched only to test if there is a next page
         ids = ids[:limit]
@@ -234,7 +234,7 @@ def handle(self) -> QueryResponse[T]:
         results = list(Session.execute(query).scalars())
 
         # Sort in the same order as returned in search response
-        results = sorted(results, key=lambda r: ids.index(r.id))
+        results = sorted(results, key=lambda r, ids=ids: ids.index(r.id))  # type: ignore
 
         response: QueryResponse[T] = {
             "total": res["estimatedTotalHits"],

backend/howtheyvote/config.py

@@ -25,7 +25,7 @@
 REQUEST_SLEEP = 0.25
 
 # Misc
-CURRENT_TERM = 9
+CURRENT_TERM = 10
 TIMEZONE = "Europe/Brussels"
 WORKER_PROMETHEUS_PORT = 3000
 SEARCH_INDEX_PREFIX = env.get("HTV_SEARCH_INDEX_PREFIX", None)

backend/howtheyvote/data/groups.json

@@ -14,9 +14,9 @@
     "label": "European People\u2019s Party",
     "short_label": "EPP",
     "alt_labels": [
+      "Group of the European People\u2019s Party (Christian Democrats) and European Democrats",
       "Group of the European People\u2019s Party (Christian Democrats)",
-      "Group of the European People\u2019s Party (Christian-Democratic Group)",
-      "Group of the European People\u2019s Party (Christian Democrats) and European Democrats"
+      "Group of the European People\u2019s Party (Christian-Democratic Group)"
     ],
     "start_date": "1979-07-17",
     "end_date": null
@@ -38,8 +38,8 @@
     "label": "The Left in the European Parliament \u2013 GUE/NGL",
     "short_label": "GUE/NGL",
     "alt_labels": [
-      "Confederal Group of the European United Left \u2013 Nordic Green Left",
-      "European United Left \u2013 Nordic Green Left"
+      "European United Left \u2013 Nordic Green Left",
+      "Confederal Group of the European United Left \u2013 Nordic Green Left"
     ],
     "start_date": "1995-01-06",
     "end_date": null
@@ -78,7 +78,9 @@
     "official_label": "Group of the Progressive Alliance of Socialists and Democrats in the European Parliament",
     "label": "Progressive Alliance of Socialists and Democrats",
     "short_label": "S&D",
-    "alt_labels": [],
+    "alt_labels": [
+      "Progressive Alliance of Socialists and Democrats in the European Parliament"
+    ],
     "start_date": "2009-07-14",
     "end_date": null
   },
@@ -107,8 +109,8 @@
     "short_label": "Rainbow Group in the European Parliament",
     "alt_labels": [
       "Rainbow",
-      "Rainbow in the European Parliament",
-      "Rainbow Group: Federation of the Green-Alternative European Links, Agelev-Ecolo, the Danish People\u2019s Movement against Membership of the European Community and the European Free Alliance in the European Parliament"
+      "Rainbow Group: Federation of the Green-Alternative European Links, Agelev-Ecolo, the Danish People\u2019s Movement against Membership of the European Community and the European Free Alliance in the European Parliament",
+      "Rainbow in the European Parliament"
     ],
     "start_date": "1984-07-24",
     "end_date": "1994-07-18"
@@ -139,8 +141,8 @@
     "label": "Christian-Democratic Group (Group of the European People\u2019s Party)",
     "short_label": "Christian-Democratic Group (Group of the European People\u2019s Party)",
     "alt_labels": [
-      "Christian-Democratics",
-      "Christian-Democratic Group"
+      "Christian-Democratic Group",
+      "Christian-Democratics"
     ],
     "start_date": "1953-06-23",
     "end_date": "1979-07-16"
@@ -152,8 +154,8 @@
     "short_label": "Communist and Allies",
     "alt_labels": [
       "Communists and Allies",
-      "Communists and Allies (SF, Ind. Sin.)",
-      "Communist and Allies Group (SF, Ind. Sin.)"
+      "Communist and Allies Group (SF, Ind. Sin.)",
+      "Communists and Allies (SF, Ind. Sin.)"
     ],
     "start_date": "1973-10-16",
     "end_date": "1989-07-24"
@@ -386,11 +388,11 @@
     "label": "Socialist Group in the European Parliament",
     "short_label": "Socialist Group in the European Parliament",
     "alt_labels": [
+      "Party of European Socialists",
       "Socialists",
+      "Socialist Group",
       "Group of the Party of European Socialists",
-      "Party of European Socialists",
-      "Socialist",
-      "Socialist Group"
+      "Socialist"
     ],
     "start_date": "1953-06-23",
     "end_date": "2009-07-13"
@@ -434,5 +436,22 @@
     ],
     "start_date": "1995-07-05",
     "end_date": "1999-07-19"
+  },
+  {
+    "code": "ESN",
+    "official_label": "Europe of Sovereign Nations Group",
+    "label": "Europe of Sovereign Nations",
+    "short_label": "ESN",
+    "start_date": "2024-07-10",
+    "alt_labels": []
+
+  },
+  {
+    "code": "PFE",
+    "official_label": "Patriots for Europe Group",
+    "label": "Patriots for Europe",
+    "short_label": "PFE",
+    "start_date": "2024-07-08",
+    "alt_labels": []
   }
 ]

backend/howtheyvote/scrapers/members.py

@@ -199,6 +199,7 @@ def _group(self, tag: Tag) -> str:
             "Treasurer",
             "Co-treasurer",
             "Member",
+            "First Vice-Chair/Member of the Bureau",
         ]
 
         for pos in positions: