Save full Firefox profile

docs/Configuration.md

@@ -46,7 +46,7 @@ of configurations of `class<BrowserParams>`.
 - `data_directory`
   - The directory into which screenshots and page dumps will be saved
   - [Intended to be removed by #232](https://github.com/mozilla/OpenWPM/issues/232)
-- `log_directory` -> supported file extensions are `.log`
+- `log_path` -> supported file extensions are `.log`
   - The path to the file in which OpenWPM will log. The
     directory given will be created if it does not exist.
 - `failure_limit` -> has to be either of type `int` or `None`
@@ -287,17 +287,7 @@ browser before visiting the next `site` in `sites`.
 
 ### Loading and saving a browser profile
 
-It's possible to load and save profiles during stateful crawls. Profile dumps
-currently consist of the following browser storage items:
-
-- cookies
-- localStorage
-- IndexedDB
-- browser history
-
-Other browser state, such as the browser cache, is not saved. In
-[Issue #62](https://github.com/citp/OpenWPM/issues/62) we plan to expand
-profiles to include all browser storage.
+It's possible to load and save profiles during stateful crawls.
 
 #### Save a profile
 

openwpm/browser_manager.py

@@ -674,6 +674,7 @@ def _start_extension(self, browser_profile_path: Path) -> ClientSocket:
             with open(ep_filename, "rt") as f:
                 port = int(f.read().strip())
 
+        ep_filename.unlink()
         self.logger.debug(
             "BROWSER %i: Connecting to extension on port %i"
             % (self.browser_params.browser_id, port)

openwpm/commands/profile_commands.py

@@ -1,5 +1,4 @@
 import logging
-import shutil
 import tarfile
 from pathlib import Path
 
@@ -47,47 +46,22 @@ def dump_profile(
         % (browser_params.browser_id, browser_profile_path, tar_path)
     )
 
-    storage_vector_files = [
+    tar.add(browser_profile_path, arcname="")
+    archived_items = tar.getnames()
+    tar.close()
+
+    required_items = [
         "cookies.sqlite",  # cookies
-        "cookies.sqlite-shm",
-        "cookies.sqlite-wal",
         "places.sqlite",  # history
-        "places.sqlite-shm",
-        "places.sqlite-wal",
         "webappsstore.sqlite",  # localStorage
-        "webappsstore.sqlite-shm",
-        "webappsstore.sqlite-wal",
-    ]
-    storage_vector_dirs = [
-        "webapps",  # related to localStorage?
-        "storage",  # directory for IndexedDB
     ]
-    for item in storage_vector_files:
-        full_path = browser_profile_path / item
-        if (
-            not full_path.is_file()
-            and not full_path.name.endswith("shm")
-            and not full_path.name.endswith("wal")
-        ):
+    for item in required_items:
+        if item not in archived_items:
             logger.critical(
-                "BROWSER %i: %s NOT FOUND IN profile folder, skipping."
-                % (browser_params.browser_id, full_path)
+                "BROWSER %i: %s NOT FOUND IN profile folder"
+                % (browser_params.browser_id, item)
             )
-        elif not full_path.is_file() and (
-            full_path.name.endswith("shm") or full_path.name.endswith("wal")
-        ):
-            continue  # These are just checkpoint files
-        tar.add(full_path, arcname=item)
-    for item in storage_vector_dirs:
-        full_path = browser_profile_path / item
-        if not full_path.is_dir():
-            logger.warning(
-                "BROWSER %i: %s NOT FOUND IN profile folder, skipping."
-                % (browser_params.browser_id, full_path)
-            )
-            continue
-        tar.add(full_path, arcname=item)
-    tar.close()
+            raise RuntimeError("Profile dump not successful")
 
 
 class DumpProfileCommand(BaseCommand):

test/test_profile.py

@@ -1,3 +1,4 @@
+import tarfile
 from pathlib import Path
 from typing import Any
 
@@ -22,7 +23,36 @@ def test_saving(default_params, task_manager_creator):
     manager, _ = task_manager_creator((manager_params, browser_params[:1]))
     manager.get(BASE_TEST_URL)
     manager.close()
-    assert (browser_params[0].profile_archive_dir / "profile.tar.gz").is_file()
+    tar_path = browser_params[0].profile_archive_dir / "profile.tar.gz"
+    assert tar_path.is_file()
+    # Test that the archived profile contains some basic items
+    profile_items = [
+        "cookies.sqlite",
+        "places.sqlite",
+        "webappsstore.sqlite",
+        "prefs.js",
+        "bookmarkbackups",
+        "cache2",
+        "storage",
+    ]
+    with tarfile.open(tar_path, "r:gz") as tar:
+        archive_items = tar.getnames()
+    for item in profile_items:
+        assert item in archive_items
+
+
+def test_save_incomplete_profile_error(default_params, task_manager_creator):
+    manager_params, browser_params = default_params
+    manager_params.num_browsers = 1
+    browser_params[0].profile_archive_dir = (
+        manager_params.data_directory / "browser_profile"
+    )
+    manager, _ = task_manager_creator((manager_params, browser_params[:1]))
+    manager.get(BASE_TEST_URL)
+    (manager.browsers[0].current_profile_path / "cookies.sqlite").unlink()
+    with pytest.raises(RuntimeError) as error:
+        manager.close()
+    assert str(error.value) == "Profile dump not successful"
 
 
 def test_crash_profile(default_params, task_manager_creator):