Format submitted URLs with PreparedRequest.prepare_url instead of requests.utils.requote_uri

docker-compose.override.yml

@@ -48,7 +48,7 @@ services:
       - scoop_rest_api_internal
 
   scoop-rest-api:
-    image: registry.lil.tools/harvardlil/scoop-rest-api:49-ede02b53aea1e3288ba0f892844dbfe0
+    image: registry.lil.tools/harvardlil/scoop-rest-api:50-395846bdca4194fadbc79ff86b1a1cf4
     init: true
     tty: true
     depends_on:

perma_web/api/views.py

@@ -511,7 +511,7 @@ def post(self, request, format=None):
                     role='primary',
                     status='pending',
                     record_type='response',
-                    url=link.submitted_url,
+                    url=link.ascii_safe_url,
                 ).save()
 
                 # create screenshot placeholder

perma_web/perma/models.py

@@ -1599,8 +1599,16 @@ def ia_identifier(self):
 
     @cached_property
     def ascii_safe_url(self):
-        """ Encoded URL as string rather than unicode. """
-        return requests.utils.requote_uri(self.submitted_url)
+        """URL as encoded internally by python requests"""
+        try:
+            # Attempt to quote the URL as well as possible:
+            # - percent encoding
+            # - unicode domains to punycode
+            # - etc.
+            return requests.Request('GET', self.submitted_url).prepare().url
+        except requests.exceptions.RequestException:
+            # If that fails, just percent encode everything for safety
+            return requests.utils.requote_uri(self.submitted_url)
 
     @cached_property
     def url_details(self):

services/docker/scoop-rest-api/config.py

@@ -1,5 +1,5 @@
-# This is the default config.py from the Scoop REST API as of 11/1/2023
-# https://github.com/harvard-lil/scoop-rest-api/blob/26dfc224aafabb53b4af5a44ef9b29cd79d1de82/scoop_rest_api/config.py
+# This is the default config.py from the Scoop REST API as of 12/5/2023
+# https://github.com/harvard-lil/perma-scoop-api/blob/709d9a96a904698143c811989e37ce91a4265448/scoop_rest_api/config.py
 # We only use it to override the blocklist: we disable it to allow the capturing of
 # localhost in our test suite.
 
@@ -80,6 +80,7 @@
 TEMPORARY_STORAGE_EXPIRATION = 60 * 60 * 24
 """ How long should temporary files be stored for? (In seconds). Can be provided via an environment variable. """  # noqa
 
+DEPLOYMENT_SENTINEL_PATH = "/tmp/deployment-pending"
 
 #
 # API-wide settings
@@ -152,5 +153,5 @@
     - utils.config_check.EXCLUDED_SCOOP_CLI_OPTIONS
 """
 
-SCOOP_TIMEOUT_FUSE = 30
+SCOOP_TIMEOUT_FUSE = 35
 """ Number of seconds to wait before "killing" a Scoop progress after capture timeout. """