From db2431e9be584510fb2ecb4b9990a6255985583e Mon Sep 17 00:00:00 2001
From: Katie Rischpater <98350084+the-bay-kay@users.noreply.github.com>
Date: Thu, 21 Dec 2023 15:48:53 -0800
Subject: [PATCH] Updated data loading instructions

Expanded upon data loading instructions, added a docker.yml and
load_mongodump.sh file to assist in the loading process.
---
 README.md              | 20 +++++++++++++++-----
 bin/load_mongodump.sh  |  9 +++++++++
 docker-compose.dev.yml | 17 +++++++++++++++++
 3 files changed, 41 insertions(+), 5 deletions(-)
 create mode 100644 bin/load_mongodump.sh
 create mode 100644 docker-compose.dev.yml

diff --git a/README.md b/README.md
index 9d6347cc..0ea5c8cd 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ it needs to be included while running them.
 
 1. Set the home environment variable
 
-    ```
+    ```bash
     $ export EMISSION_SERVER_HOME=<path_to_emission_server_repo>
     ```
 
@@ -21,26 +21,26 @@ it needs to be included while running them.
     
 1. If you haven't setup before, set up the evaluation system
 
-    ```
+    ```bash
     $ source setup.sh
     ```
 1. If you have, activate
 
-    ```
+    ```bash
     $ source activate.sh
     ```
 
 1. Access the visualizations of interest and copy the config over. The `<eval_folder>` mentioned below can be any folder containing notebooks and/or .py files for visualisation or other purposes. E.g. : `TRB_label_assist` is one such folder.
 
 
-```
+```bash
 $ cd <eval_folder>
 $ cp -r ../conf .
 ```
 
 1. Start the notebook server
 
-```
+```bash
 $ ../bin/em-jupyter-notebook.sh
 ```
 
@@ -50,6 +50,16 @@ $ ../bin/em-jupyter-notebook.sh
   the top of the notebook, and request the data for research purposes using 
     https://github.com/e-mission/e-mission-server/wiki/Requesting-data-as-a-collaborator
 
+- Assuming that your data is in the "mongodump" format, this repository has a helper script to load the data directly into the database.  
+    - Navigate to the `e-mission-eval-private-data/` directory and start the docker environment
+        ```bash
+            $ docker-compose -f docker-compose.dev.yml up
+        ```
+    - In another terminal, again navigate to the repository. Using the script provided, load the mongodump into docker
+        ```bash
+        $ bash bin/load_mongodump.sh <mongodump_file.tar.gz>
+        ```
+    - Depending on the size of the mongodump, the loading step may take quite a long time (up to _several hours_).  For more details on how to speed up this process, please refer to the data request documentation [here](https://github.com/e-mission/e-mission-server/wiki/Requesting-data-as-a-collaborator).
 ### Cleaning up
 
 After completing analysis, tear down
diff --git a/bin/load_mongodump.sh b/bin/load_mongodump.sh
new file mode 100644
index 00000000..d131cf15
--- /dev/null
+++ b/bin/load_mongodump.sh
@@ -0,0 +1,9 @@
+MONGODUMP_FILE=$1
+
+echo "Copying file to docker container"
+docker cp $MONGODUMP_FILE e-mission-eval-private-data-db-1:/tmp
+
+FILE_NAME=`basename $MONGODUMP_FILE`
+
+echo "Restoring the dump from $FILE_NAME"
+docker exec -e MONGODUMP_FILE=$FILE_NAME e-mission-eval-private-data-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore'
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
new file mode 100644
index 00000000..476524d4
--- /dev/null
+++ b/docker-compose.dev.yml
@@ -0,0 +1,17 @@
+version: "3"
+services:
+  db:
+    image: mongo:4.4.0
+    volumes:
+      - mongo-data:/data/db
+    networks:
+       - emission
+    ports:
+      # 
+      - "27017:27017"
+
+networks:
+  emission:
+
+volumes:
+  mongo-data: