higlass · Jan 8, 2020 · Jan 9, 2020 · Jan 11, 2020 · Jan 11, 2020 · Jan 13, 2020
Showing with 385 additions and 26 deletions.

+5 −5 .travis.yml

+25 −0 README.md

+52 −0 get_test_data.sh

+2 −0 higlass_manage/cli.py

+33 −0 higlass_manage/common.py

+59 −21 higlass_manage/stop.py

+194 −0 higlass_manage/update_viewconfs.py

+15 −0 test.sh
diff --git a/.travis.yml b/.travis.yml
@@ -2,25 +2,25 @@
 # Tweaked to specify versions on everything for stability.
 language: python
 python:
-  - "3.6"
+  - '3.6'
 
 cache: apt
 
 services:
-    - docker
+  - docker
 
 before_install:
   - wget http://repo.continuum.io/miniconda/Miniconda3-4.3.21-Linux-x86_64.sh -O miniconda.sh
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
+  - conda update --yes conda
+  - conda install -y -c bioconda pysam
   # Prefer stability to having the latest
-  #- conda update --yes conda
-
 
 install:
   - conda install -c conda-forge --yes python=$TRAVIS_PYTHON_VERSION scipy
   - pip install -r requirements.txt
   - python setup.py install
 
 script:
-  - ./test.sh
+  - ./test.sh
diff --git a/README.md b/README.md
@@ -138,6 +138,31 @@ To stop a running instance, use the `stop` command:
 higlass-manage stop
 ```
 
+### Migrating a HiGlass instance
+
+Migrating a higlass instance between different servers can be done by copying the data-folder, typically `hg-data`, from server of origin to the destination and re-starting higlass:
+```bash
+# at the destination:
+scp -r user@old.host.org:/path/to/hg-data  /new/path/
+higlass-manage start --data-dir /new/path/hg-data ...
+```
+Tilesets ingested at the origin would be available at the destination. However, `viewconf`-s saved at the origin would not work at the destination, because the tilesets would be referred there with original URLs, e.g. `http://old.server.org:PORT`.
+
+This can be fixed by updating `viewconfs` in the database before copying `hg-data`:
+```bash
+# at the old.host.org:
+higlass-manage update-viewconfs --hg-name-old old_hg_name --new-site-url http://new.host.org
+```
+in this case, higlass instance `old_hg_name` would be used to infer old site URL, port and path to the data folder.
+
+Same can be achieved even without any running higlass instances, but then one has to provide path to the data folder and site's URL and port both "new" and "old":
+```bash
+# at the old.host.org:
+higlass-manage update-viewconfs --old-site-url http://old.host.org --data-dir /old/path/to/data --new-site-url http://new.host.org
+```
+`update-viewconfs` would save updated database as `/old/path/to/data/db.sqlite3.updated` and keep the original `/old/path/to/data/db.sqlite3` unchanged. Thus, `db.sqlite3.updated` has to be renamed to `db.sqlite3` after migrating to `new.host.org`.
+
+
 ## Development
 
 The following is a list of handy commands when developing HiGlass:

diff --git a/get_test_data.sh b/get_test_data.sh
@@ -13,3 +13,55 @@ for FILE in $FILES; do
   [ -e data/$FILE ] || wget -P data/ https://s3.amazonaws.com/pkerp/public/$FILE
 done
 
+
+VIEWCONF=$(cat <<EOF
+{
+  "uid": "test-123",
+  "viewconf": {
+      "editable": true,
+      "views": [
+        {
+          "uid": "myviewconf123",
+          "tracks": {
+            "top": [],
+            "center": [
+              {
+                "type": "heatmap",
+                "options": {
+                  "valueScaleMax": 0.2
+                },
+                "tilesetUid": "a",
+                "server": "http://localhost:8123/api/v1/",
+                "height": 250
+              }
+            ],
+            "left": [],
+            "right": [],
+            "bottom": []
+          },
+          "layout": {
+            "w": 12,
+            "h": 6,
+            "x": 0,
+            "y": 0
+          }
+        }
+      ],
+      "trackSourceServers": [
+        "http://localhost:8123/api/v1/"
+      ],
+      "locationLocks": {
+        "locksByViewUid": {},
+        "locksDict": {}
+      },
+      "zoomLocks": {
+        "locksByViewUid": {},
+        "locksDict": {}
+      },
+      "exportViewUrl": "http://localhost:8123/api/v1/viewconfs"
+    }
+}
+EOF
+)
+
+echo $VIEWCONF > data/test_viewconf.json
diff --git a/higlass_manage/cli.py b/higlass_manage/cli.py
@@ -6,6 +6,7 @@
 from higlass_manage.stop import stop
 from higlass_manage.shell import shell
 from higlass_manage.view import view
+from higlass_manage.update_viewconfs import update_viewconfs
 from higlass_manage.logs import logs
 from higlass_manage import __version__
 
@@ -56,5 +57,6 @@ def cli():
 cli.add_command(stop)
 cli.add_command(shell)
 cli.add_command(view)
+cli.add_command(update_viewconfs)
 cli.add_command(logs)
 cli.add_command(version)
diff --git a/higlass_manage/common.py b/higlass_manage/common.py
@@ -9,6 +9,7 @@
 NETWORK_PREFIX = "higlass-manage-network"
 REDIS_PREFIX = "higlass-manage-redis"
 REDIS_CONF = "/usr/local/etc/redis/redis.conf"
+SQLITEDB = "db.sqlite3"
 
 
 def md5(fname):
@@ -41,6 +42,38 @@ def get_port(hg_name):
     return port
 
 
+def get_site_url(hg_name, _SITE_URL="SITE_URL"):
+    """
+    get SITE_URL for a given container
+    using container docker-config, assuming
+    there is not more than one SITE_URL in
+    container's env.
+
+    Yields "localhost" when no SITE_URL entries
+    detected.
+    """
+
+    client = docker.from_env()
+
+    container_name = hg_name_to_container_name(hg_name)
+    config = client.api.inspect_container(container_name)
+
+    site_url_entries = [s for s in config["Config"]["Env"] if _SITE_URL in s]
+    # if there is no SITE_URL entry yield "localhost"
+    if not site_url_entries:
+        return "http://localhost"
+    # otherwise there has to be only one SITE_URL entry in the Env:
+    elif len(site_url_entries) > 1:
+        raise ValueError(
+            "There are multiple SITE_URL entry in {} env".format(container_name)
+        )
+    else:
+        (site_url,) = site_url_entries
+        # parse "SITE_URL=http://hostname":
+        _, site_url = site_url.split("=")
+        return site_url
+
+
 def fill_filetype_and_datatype(filename, filetype, datatype):
     """
     If no filetype or datatype are provided, add them

diff --git a/higlass_manage/stop.py b/higlass_manage/stop.py
@@ -7,9 +7,44 @@
 
 @click.command()
 @click.argument("names", nargs=-1)
-def stop(names):
+@click.option(
+    "--remove-container/--dont-remove-container",
+    default=True,
+    show_default=True,
+    help="Remove stopped higlass container",
+)
+@click.option(
+    "--stop-redis/--dont-stop-redis",
+    default=True,
+    show_default=True,
+    help="Stop and remove redis container"
+    " associated with a given higlass"
+    " instance.",
+)
+@click.option(
+    "--remove-network-bridge/--dont-remove-network-bridge",
+    default=True,
+    show_default=True,
+    help="Remove network bridge associated with a given higlass instance.",
+)
+def stop(
+    names, remove_container, stop_redis, remove_network_bridge,
+):
+    _stop(
+        names, remove_container, stop_redis, remove_network_bridge,
+    )
+
+
+def _stop(
+    names, remove_container=True, stop_redis=True, remove_network_bridge=True,
+):
     """
-    Stop a running instance
+    Stop a running higlass instance along with the
+    associated redis container and network bridges.
+
+    The script attemps to stop and remove all of the
+    containers/networks associated with a given higlass
+    name.
     """
     client = docker.from_env()
 
@@ -21,28 +56,31 @@ def stop(names):
         hm_name = "{}-{}".format(CONTAINER_PREFIX, name)
         try:
             client.containers.get(hm_name).stop()
-            client.containers.get(hm_name).remove()
+            if remove_container:
+                client.containers.get(hm_name).remove()
         except docker.errors.NotFound as ex:
             sys.stderr.write("Instance not running: {}\n".format(name))
 
         # redis container
-        redis_name = "{}-{}".format(REDIS_PREFIX, name)
-        try:
-            client.containers.get(redis_name).stop()
-            client.containers.get(redis_name).remove()
-        except docker.errors.NotFound:
-            sys.stderr.write(
-                "No Redis instances found at {}; skipping...\n".format(redis_name)
-            )
+        if stop_redis:
+            redis_name = "{}-{}".format(REDIS_PREFIX, name)
+            try:
+                client.containers.get(redis_name).stop()
+                client.containers.get(redis_name).remove()
+            except docker.errors.NotFound:
+                sys.stderr.write(
+                    "No Redis instances found at {}; skipping...\n".format(redis_name)
+                )
 
         # bridge network
-        network_name = "{}-{}".format(NETWORK_PREFIX, name)
-        try:
-            network_list = client.networks.list(names=[network_name])
-            if network_list:
-                network = client.networks.get(network_name)
-                network.remove()
-        except docker.errors.NotFound:
-            sys.stderr.write(
-                "No bridge network found at {}; skipping...\n".format(network_name)
-            )
+        if remove_network_bridge:
+            network_name = "{}-{}".format(NETWORK_PREFIX, name)
+            try:
+                network_list = client.networks.list(names=[network_name])
+                if network_list:
+                    network = client.networks.get(network_name)
+                    network.remove()
+            except docker.errors.NotFound:
+                sys.stderr.write(
+                    "No bridge network found at {}; skipping...\n".format(network_name)
+                )
diff --git a/higlass_manage/update_viewconfs.py b/higlass_manage/update_viewconfs.py
@@ -0,0 +1,194 @@
+import sys
+import click
+import os.path as op
+import sqlite3
+import shutil
+import docker
+
+from .common import (
+    get_data_dir,
+    get_site_url,
+    get_port,
+    hg_name_to_container_name,
+    SQLITEDB,
+)
+
+from .stop import _stop
+
+
+@click.command()
+@click.option(
+    "--old-hg-name",
+    help="The name of the running higlass container that needs to be updated.",
+    required=False,
+)
+@click.option(
+    "--old-site-url",
+    help="site-url at the old location."
+    " Provide this when higlass container"
+    " to be updated is not running.",
+    required=False,
+)
+@click.option(
+    "--old-port",
+    help="port at the old location."
+    " Provide this when higlass container"
+    " to be updated is not running.",
+    required=False,
+    default="80",
+    type=str,
+)
+@click.option(
+    "--old-data-dir",
+    help="data directory of the higlass"
+    " that is to be updated (usually 'hg-data')."
+    " Provide this when higlass container"
+    " is not running.",
+    required=False,
+)
+@click.option(
+    "--new-site-url",
+    default="http://localhost",
+    help="site-url at the new location.",
+    required=True,
+)
+@click.option(
+    "--new-port",
+    help="port at the new location",
+    required=False,
+    default="80",
+    type=str,
+)
+@click.option(
+    "--db-backup-name",
+    help="name of the database (db) backup file."
+    " db backup will be stored in the data directory"
+    " provided explicitly or inferred from the running"
+    " container.",
+    required=False,
+    default=f"{SQLITEDB}.updated",
+    type=str,
+)
+def update_viewconfs(
+    old_hg_name,
+    old_site_url,
+    old_port,
+    old_data_dir,
+    new_site_url,
+    new_port,
+    db_backup_name,
+):
+    """
+    The script allows one to update viewconfs saved
+    in an existing higlass database. It does so
+    by modifying references to tilesets that use
+    old-site-url:old-port --> new-site-url:new-port
+
+    old/new-site-urls must include schema (http, https):
+    http://localhost
+    http://old.host.org
+    ...
+
+    if 'old-hg-name' is provided and higlass is running,
+    then 'old-site-url,old-port,old-data-dir' are inferred.
+
+    if 'old-hg-name' is NOT provided
+    then at least 'old-site-url'and 'old-data-dir'
+    are required. This scenario should be invoked
+    only when the container is not running, otherwise
+    integrity of the DB-backup can not be guaranteed.
+
+    Post 80 is default http port and both
+    new-port and old-port defaults to it,
+    if not specified otherwise.
+    site-url:80 is equivalent to site-url
+
+    Script keeps existing database unchanged,
+    but modifies a backed up version located
+    in the same path as the original one.
+
+    Running higlass-container would be interrupted
+    during database backup.
+
+    """
+
+    # update viewconfs FROM (ORIGIN):
+    if old_hg_name is not None:
+        # then the container must be running
+        try:
+            old_site_url = get_site_url(old_hg_name)
+            old_port = get_port(old_hg_name)
+            old_data_dir = get_data_dir(old_hg_name)
+        except docker.errors.NotFound as ex:
+            sys.stderr.write(f"Instance not running: {old_hg_name}\n")
+    elif (old_site_url is None) or (old_data_dir is None):
+        raise ValueError(
+            "old-site-url and old-data-dir must be provided,"
+            " when instance is not running and no old-hg-name is provided\n"
+        )
+
+    # define origin as site_url:port or site_url (when 80)
+    origin = old_site_url if (old_port == "80") else f"{old_site_url}:{old_port}"
+
+    # update viewconfs TO (DESTINATION):
+    # define destination as site_url:port or site_url (when 80)
+    destination = new_site_url if (new_port == "80") else f"{new_site_url}:{new_port}"
+
+    # locate db.sqlite3 and name for the updated version:
+    origin_db_path = op.join(old_data_dir, SQLITEDB)
+    update_db_path = op.join(old_data_dir, db_backup_name)
+
+    # backup the database using simple copyfile, stop container before
+    if old_hg_name is not None:
+        _stop([old_hg_name,], False, False, False)
+    try:
+        shutil.copyfile(origin_db_path, update_db_path)
+    except (OSError, IOError):
+        sys.stderr.write(f"Failed to copy {origin_db_path} to {update_db_path}")
+        sys.exit(-1)
+    finally:
+        # restart container even if exception occurs:
+        if old_hg_name is not None:
+            sys.stderr.write(f"Restarting container {old_hg_name} ...\n")
+            sys.stderr.flush()
+            client = docker.from_env()
+            container_name = hg_name_to_container_name(old_hg_name)
+            client.containers.get(container_name).restart()
+    # alternatively database could be backed up using a "backup"-mechanism:
+    # CLI
+    #    res = subprocess.run(["sqlite3", origin_db_path, f".backup {update_db_path}"])
+    # Python API >= 3.7.0
+    #       conn = sqlite3.connect(origin_db_path)
+    #       conn.backup(update_db_path)
+    # check if it is indeed "atomic" and can be done on a "live" database .
+
+    # now modify the backed-up database "update_db_path" using sqlite3 API:
+    conn = None
+    try:
+        conn = sqlite3.connect(update_db_path)
+    except sqlite3.Error as e:
+        sys.stderr.write(f"Failed to connect to {update_db_path}\n")
+        sys.exit(-1)
+
+    # sql query to update viewconfs, by replacing origin -> destination
+    db_query = f"""
+            UPDATE tilesets_viewconf
+            SET viewconf = replace(viewconf,'{origin}','{destination}')
+            """
+    # exec sql query
+    with conn:
+        cur = conn.cursor()
+        cur.execute(db_query)
+
+    conn.close()
+    # todo: add some stats on - how many viewconfs
+    # were updated
+
+    sys.stderr.write(
+        f"Backed up version of the database {update_db_path}\n"
+        " has been updated and ready for migration\n\n"
+        " copy it to the new host along with the media folder\n"
+        f" rename the database file back to {SQLITEDB} and restart higlass.\n"
+    )
+    sys.stderr.flush()
+    sys.exit(0)
diff --git a/test.sh b/test.sh
@@ -62,6 +62,21 @@ start wait
         || die
 end wait
 
+
+start update-viewconfs
+    # if I understand correctly "test-hg" is still alive by now !
+    # so we'll use it to test "update-viewconfs":
+    URL="http://localhost:$PORT/api/v1/viewconfs/"
+    # 1) post viewconf - to simulate a viewconf saved as a link ...
+    curl -d "@data/test_viewconf.json" -X POST ${URL}
+    # 2) once "test-123" viewconf is on server-side, i.e. in the "${TMPDIR}/test-hg-data/db.sqlite3"
+    higlass-manage update-viewconfs --old-hg-name test-hg --new-site-url new.host.org
+    # 3) now "${TMPDIR}/test-hg-data/db.sqlite3.updated" should become available, with new.host.org instead
+    sqlite3 ${TMPDIR}/test-hg-data/db.sqlite3.updated \
+        "SELECT viewconf FROM tilesets_viewconf WHERE uuid='test-123'" | grep new.host.org \
+        || die
+end update-viewconfs
+
 start cleanup
     higlass-manage stop test-hg
 end cleanup