Scoop logging

perma_web/perma/celery_tasks.py

@@ -1063,7 +1063,15 @@ def capture_with_scoop(capture_job):
             link.primary_capture.status = 'success'
             link.primary_capture.save(update_fields=['status'])
         else:
-            logger.error(f"Scoop capture failed: {poll_data}")
+            didnt_load = "ERROR Navigation to page failed (about:blank)"
+            if poll_data['stderr_logs'] and didnt_load in poll_data['stderr_logs']:
+                logger.warning(f"{capture_job.link_id}: Scoop failed to load submitted URL ({capture_job.submitted_url}).")
+                capture_job.link.tags.add('scoop-load-failure')
+            elif not poll_data['stderr_logs'] and not poll_data['stdout_logs']:
+                logger.warning(f"{capture_job.link_id}: Scoop failed without logs ({poll_data['id_capture']}).")
+                capture_job.link.tags.add('scoop-silent-failure')
+            else:
+                logger.error(f"Scoop capture failed: {poll_data}")
 
     except HaltCaptureException:
         print("HaltCaptureException thrown")

perma_web/requirements.txt

@@ -1168,31 +1168,34 @@ warcprox==2.4.17 \
 warctools==4.10.0 \
     --hash=sha256:ce0c6e274db8ac8810f7c97b3943e8e8deadbc3f5c982db77cddaae2d2ae6170
     # via warcprox
-watchdog==2.1.7 \
-    --hash=sha256:03b43d583df0f18782a0431b6e9e9965c5b3f7cf8ec36a00b930def67942c385 \
-    --hash=sha256:0908bb50f6f7de54d5d31ec3da1654cb7287c6b87bce371954561e6de379d690 \
-    --hash=sha256:0b4a1fe6201c6e5a1926f5767b8664b45f0fcb429b62564a41f490ff1ce1dc7a \
-    --hash=sha256:177bae28ca723bc00846466016d34f8c1d6a621383b6caca86745918d55c7383 \
-    --hash=sha256:19b36d436578eb437e029c6b838e732ed08054956366f6dd11875434a62d2b99 \
-    --hash=sha256:1d1cf7dfd747dec519486a98ef16097e6c480934ef115b16f18adb341df747a4 \
-    --hash=sha256:1e877c70245424b06c41ac258023ea4bd0c8e4ff15d7c1368f17cd0ae6e351dd \
-    --hash=sha256:340b875aecf4b0e6672076a6f05cfce6686935559bb6d34cebedee04126a9566 \
-    --hash=sha256:351e09b6d9374d5bcb947e6ac47a608ec25b9d70583e9db00b2fcdb97b00b572 \
-    --hash=sha256:3fd47815353be9c44eebc94cc28fe26b2b0c5bd889dafc4a5a7cbdf924143480 \
-    --hash=sha256:49639865e3db4be032a96695c98ac09eed39bbb43fe876bb217da8f8101689a6 \
-    --hash=sha256:4d0e98ac2e8dd803a56f4e10438b33a2d40390a72750cff4939b4b274e7906fa \
-    --hash=sha256:6e6ae29b72977f2e1ee3d0b760d7ee47896cb53e831cbeede3e64485e5633cc8 \
-    --hash=sha256:7f14ce6adea2af1bba495acdde0e510aecaeb13b33f7bd2f6324e551b26688ca \
-    --hash=sha256:81982c7884aac75017a6ecc72f1a4fedbae04181a8665a34afce9539fc1b3fab \
-    --hash=sha256:81a5861d0158a7e55fe149335fb2bbfa6f48cbcbd149b52dbe2cd9a544034bbd \
-    --hash=sha256:ae934e34c11aa8296c18f70bf66ed60e9870fcdb4cc19129a04ca83ab23e7055 \
-    --hash=sha256:b26e13e8008dcaea6a909e91d39b629a39635d1a8a7239dd35327c74f4388601 \
-    --hash=sha256:b3750ee5399e6e9c69eae8b125092b871ee9e2fcbd657a92747aea28f9056a5c \
-    --hash=sha256:b61acffaf5cd5d664af555c0850f9747cc5f2baf71e54bbac164c58398d6ca7b \
-    --hash=sha256:b9777664848160449e5b4260e0b7bc1ae0f6f4992a8b285db4ec1ef119ffa0e2 \
-    --hash=sha256:bdcbf75580bf4b960fb659bbccd00123d83119619195f42d721e002c1621602f \
-    --hash=sha256:d802d65262a560278cf1a65ef7cae4e2bc7ecfe19e5451349e4c67e23c9dc420 \
-    --hash=sha256:ed6d9aad09a2a948572224663ab00f8975fae242aa540509737bb4507133fa2d
+watchdog==3.0.0 \
+    --hash=sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a \
+    --hash=sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100 \
+    --hash=sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8 \
+    --hash=sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc \
+    --hash=sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae \
+    --hash=sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41 \
+    --hash=sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0 \
+    --hash=sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f \
+    --hash=sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c \
+    --hash=sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9 \
+    --hash=sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3 \
+    --hash=sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709 \
+    --hash=sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83 \
+    --hash=sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759 \
+    --hash=sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9 \
+    --hash=sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3 \
+    --hash=sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7 \
+    --hash=sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f \
+    --hash=sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346 \
+    --hash=sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674 \
+    --hash=sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397 \
+    --hash=sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96 \
+    --hash=sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d \
+    --hash=sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a \
+    --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \
+    --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \
+    --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33
     # via -r requirements.in
 wcwidth==0.2.5 \
     --hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \