Merge pull request #51 from hubmapconsortium/devel-based-1.8.0

Devel based 1.8.0

.gitignore

@@ -1,8 +1,14 @@
 # __pycache__ directories in any path
 **/__pycache__
 
+BUILD
+
+# the copied
+hubmap-auth/src/BUILD
+hubmap-auth/src/VERSION
+
 # log files in any path
 **/*.log
 
 # hubmap-auth app.cfg
-hubmap-auth/src/instance/app.cfg
+hubmap-auth/src/instance/app.cfg

README.md

@@ -66,7 +66,7 @@ The Docker daemon binds to a Unix socket instead of a TCP port. By default that
 sudo usermod -aG docker $USER
 ````
 
-The log out and log back in so that your group membership is re-evaluated. If testing on a virtual machine, it may be necessary to restart the virtual machine for changes to take effect.
+Then log out and log back in so that your group membership is re-evaluated. If testing on a virtual machine, it may be necessary to restart the virtual machine for changes to take effect.
 
 Note: the following instructions with docker commands are based on managing Docker as a non-root user.
 
@@ -112,13 +112,12 @@ To see the usage of this script:
 It outputs:
 
 ````
-Usage: ./hubmap-docker.sh [-vhN] [localhost|dev|test|stage|prod] [build|start|stop|check|config]
+Usage: ./hubmap-docker.sh [-vh] [localhost|dev|test|stage|prod] [build|start|stop|check|config]
        -v verbose
        -h help
-       -N use --no-cache for build
 ````
 
-The `hubmap-docker.sh` basically takes two arguments: deployment environment (localhost|dev|test|stage|prod) and the option (build|start|stop|check|config). In addition, you can also use `-v` to see the verbose output and `-h` for the usage help tip. The `--no-cache` is used with `build` to avoid the docker cache when creating images.
+The `hubmap-docker.sh` basically takes two arguments: deployment environment (localhost|dev|test|stage|prod) and the option (build|start|stop|check|config). In addition, you can also use `-v` to see the verbose output and `-h` for the usage help tip. 
 
 Before we go ahead to start building the docker images, we can do a check to see if all the required configuration files are in place:
 
@@ -211,13 +210,40 @@ nginx -s reload
 * Reloading is safer than restarting because if a syntax error is noticed in a config file, it will not proceed with the reload and your server remains running.
 * If there is a syntax error in a config file and you restart, it's possible the server will not restart correctly.
 
+## Rebuild `hubmap-auth` image individually
+
+You may only need to rebuild the `hubmap-auth` image while keeping other HuBMAP docker images unchanged. We first need to export the version environment variable by sourcing the script:
+
+````
+source ./hubmap-auth-docker.sh dev build
+````
+
+To start up the hubmap-auth container:
+
+````
+source ./hubmap-auth-docker.sh dev start
+````
+
+And stop the running container by:
+
+````
+source ./hubmap-auth-docker.sh dev stop
+````
+
+You can also stop the running container and remove it by:
+
+````
+source ./hubmap-auth-docker.sh dev down
+````
+
 
 ## Update base image
 
 The `entity-api`, `uuid-api`, `ingest-api`, and `hubmap-auth` docker images are based on the `hubmap/api-base-image:latest` image. If you need to update the base image, go to the `api-base-image` directory and recrerate it with:
 
 ````
-docker build -t hubmap/api-base-image:latest
+cd api-base-image
+docker build -t hubmap/api-base-image:latest .
 ````
 
 Then publish it to the DockerHub:

VERSION

@@ -0,0 +1 @@
+1.8.0

docker-compose.dev.yml

@@ -20,6 +20,9 @@ services:
       # 8443 is used by ingest-api running on another machine
       - "8443:8443" 
     volumes:
+      # Mount the VERSION file and BUILD file
+      - "./VERSION:/usr/src/app/src/VERSION"
+      - "./BUILD:/usr/src/app/src/BUILD"
       # Mount the source code to container
       - "./hubmap-auth/src/:/usr/src/app/src"
       # Mount conf.d-test to the nginx conf.d on container

docker-compose.localhost.yml

@@ -27,11 +27,15 @@ services:
       # search-api
       - "8686:8686" 
     volumes:
+      # Mount the VERSION file and BUILD file
+      - "./VERSION:/usr/src/app/src/VERSION"
+      - "./BUILD:/usr/src/app/src/BUILD"
       # Mount the source code to container
       - "./hubmap-auth/src/:/usr/src/app/src"
       # Mount conf.d-localhost to the nginx conf.d on container
       - "./nginx/conf.d-localhost:/etc/nginx/conf.d"
       # Mount the static files
-      - "/hubmap-data:/usr/src/assets"
+      - "/hubmap-data:/hubmap-data"
       # Mount the API endpoints json file for API endpoints lookup
       - "./api_endpoints.localhost.json:/usr/src/app/api_endpoints.json"
+

docker-compose.yml

@@ -10,9 +10,16 @@ services:
         # The commons github branch to be used during image build (default to master if not set or null)
         - COMMONS_BRANCH=${COMMONS_BRANCH:-master}
     # Build the image with name and tag
-    image: hubmap-auth:1.7
+    # Exit with an error message containing err if HUBMAP_AUTH_VERSION is unset or empty in the environment
+    image: hubmap-auth:${HUBMAP_AUTH_VERSION:?err}
     hostname: hubmap-auth
     container_name: hubmap-auth
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8080"]
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
     networks:
       - gateway_hubmap
     volumes:

hubmap-auth-docker.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Print a new line and the banner
+echo
+echo "==================== HUBMAP-AUTH ===================="
+
+# Set the version environment variable for the docker build
+# Version number is from the VERSION file
+# Also remove newlines and leading/trailing slashes if present in that VERSION file
+# Note: the BUILD and VERSION files are in the same dir as this script, this is different from other APIs
+function export_version() {
+    export HUBMAP_AUTH_VERSION=$(tr -d "\n\r" < VERSION | xargs)
+    echo "HUBMAP_AUTH_VERSION: $HUBMAP_AUTH_VERSION"
+}
+
+# Generate the build version based on git branch name and short commit hash and write into BUILD file
+# Note: the BUILD and VERSION files are in the same dir as this script, this is different from other APIs
+function generate_build_version() {
+    GIT_BRANCH_NAME=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p')
+    GIT_SHORT_COMMIT_HASH=$(git rev-parse --short HEAD)
+    # Clear the old BUILD version and write the new one
+    truncate -s 0 BUILD
+    # Note: echo to file appends newline
+    echo $GIT_BRANCH_NAME:$GIT_SHORT_COMMIT_HASH >> BUILD
+    # Remmove the trailing newline character
+    truncate -s -1 BUILD
+
+    echo "BUILD(git branch name:short commit hash): $GIT_BRANCH_NAME:$GIT_SHORT_COMMIT_HASH"
+}
+
+# This function sets DIR to the directory in which this script itself is found.
+# Thank you https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself                                                                      
+function get_dir_of_this_script () {
+    SCRIPT_SOURCE="${BASH_SOURCE[0]}"
+    while [ -h "$SCRIPT_SOURCE" ]; do # resolve $SCRIPT_SOURCE until the file is no longer a symlink
+        DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" >/dev/null 2>&1 && pwd )"
+        SCRIPT_SOURCE="$(readlink "$SCRIPT_SOURCE")"
+        [[ $SCRIPT_SOURCE != /* ]] && SCRIPT_SOURCE="$DIR/$SCRIPT_SOURCE" # if $SCRIPT_SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+    done
+    DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" >/dev/null 2>&1 && pwd )"
+    echo 'DIR of script:' $DIR
+}
+
+if [[ "$1" != "localhost" && "$1" != "dev" && "$1" != "test" && "$1" != "stage" && "$1" != "prod" ]]; then
+    echo "Unknown build environment '$1', specify one of the following: localhost|dev|test|stage|prod"
+else
+    if [[ "$2" != "check" && "$2" != "config" && "$2" != "build" && "$2" != "start" && "$2" != "stop" && "$2" != "down" ]]; then
+        echo "Unknown command '$2', specify one of the following: check|config|build|start|stop|down"
+    else
+        # Always show the script dir
+        get_dir_of_this_script
+
+        # Always export and show the version
+        export_version
+
+        # Always show the build in case branch changed or new commits
+        generate_build_version
+
+        # Print empty line
+        echo
+
+        if [ "$2" = "check" ]; then
+            # Bash array
+            config_paths=(
+                'hubmap-auth/src/instance/app.cfg'
+            )
+
+            for pth in "${config_paths[@]}"; do
+                if [ ! -e $pth ]; then
+                    echo "Missing file (relative path to DIR of script) :$pth"
+                    exit -1
+                fi
+            done
+
+            echo 'Checks complete, all good :)'
+        elif [ "$2" = "config" ]; then
+            docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p gateway config
+        elif [ "$2" = "build" ]; then
+            # Only mount the VERSION file and BUILD file for localhost and dev
+            # On test/stage/prod, copy the VERSION file and BUILD file to image
+            if [[ "$1" != "localhost" && "$1" != "dev" ]]; then
+                cp VERSION hubmap-auth/src
+                cp BUILD hubmap-auth/src
+            fi
+
+            docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p gateway build
+        elif [ "$2" = "start" ]; then
+            docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p gateway up -d
+        elif [ "$2" = "stop" ]; then
+            docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p gateway stop
+        elif [ "$2" = "down" ]; then
+            docker-compose -f docker-compose.yml -f docker-compose.$1.yml -p gateway down
+        fi
+    fi
+fi

hubmap-auth/src/app.py

@@ -7,6 +7,7 @@
 import functools
 import re
 import os
+from pathlib import Path
 from urllib.parse import urlparse, parse_qs
 
 # HuBMAP commons
@@ -260,12 +261,14 @@ def status_request(target_url):
 # Dict of API status data
 def get_status_data():
     # Some constants
+    GATEWAY = 'gateway'
+    VERSION = 'version'
+    BUILD = 'build'
     UUID_API = 'uuid_api'
     ENTITY_API = 'entity_api'
     INGEST_API = 'ingest_api'
     SEARCH_API = 'search_api'
     FILE_ASSETS = 'file_assets'
-
     API_AUTH = 'api_auth'
     MYSQL_CONNECTION = 'mysql_connection'
     NEO4J_CONNECTION = 'neo4j_connection'
@@ -274,8 +277,16 @@ def get_status_data():
     FILE_ASSETS_STATUS = 'file_assets_status'
 
     # All API services have api_auth status (meaning the gateway's API auth is working)
+    # We won't get other status if api_auth fails
     # Add additional API-specific status to the dict when API auth check passes
+    # Gateway version and build are parsed from VERSION and BUILD files directly
+    # instead of making API calls. So they alwasy present
     status_data = {
+        GATEWAY: {
+            # Use strip() to remove leading and trailing spaces, newlines, and tabs
+            VERSION: (Path(__file__).parent / 'VERSION').read_text().strip(),
+            BUILD: (Path(__file__).parent / 'BUILD').read_text().strip()
+        },
         UUID_API: {
             API_AUTH: False
         },
@@ -301,6 +312,14 @@ def get_status_data():
 
         # Then parse the response json to determine if neo4j connection is working
         response_json = uuid_api_response.json()
+        if VERSION in response_json:
+            # Set version
+            status_data[UUID_API][VERSION] = response_json[VERSION]
+
+        if BUILD in response_json:
+            # Set build
+            status_data[UUID_API][BUILD] = response_json[BUILD]
+
         if MYSQL_CONNECTION in response_json:
             # Add the mysql connection status
             status_data[UUID_API][MYSQL_CONNECTION] = response_json[MYSQL_CONNECTION]
@@ -313,6 +332,14 @@ def get_status_data():
 
         # Then parse the response json to determine if neo4j connection is working
         response_json = entity_api_response.json()
+        if VERSION in response_json:
+            # Set version
+            status_data[ENTITY_API][VERSION] = response_json[VERSION]
+
+        if BUILD in response_json:
+            # Set build
+            status_data[ENTITY_API][BUILD] = response_json[BUILD]
+
         if NEO4J_CONNECTION in response_json:
             # Add the neo4j connection status
             status_data[ENTITY_API][NEO4J_CONNECTION] = response_json[NEO4J_CONNECTION]
@@ -325,6 +352,14 @@ def get_status_data():
 
         # Then parse the response json to determine if neo4j connection is working
         response_json = ingest_api_response.json()
+        if VERSION in response_json:
+            # Set version
+            status_data[INGEST_API][VERSION] = response_json[VERSION]
+
+        if BUILD in response_json:
+            # Set build
+            status_data[INGEST_API][BUILD] = response_json[BUILD]
+
         if NEO4J_CONNECTION in response_json:
             # Add the neo4j connection status
             status_data[INGEST_API][NEO4J_CONNECTION] = response_json[NEO4J_CONNECTION]
@@ -337,6 +372,14 @@ def get_status_data():
 
         # Then parse the response json to determine if elasticsearch cluster is connected
         response_json = search_api_response.json()
+        if VERSION in response_json:
+            # Set version
+            status_data[SEARCH_API][VERSION] = response_json[VERSION]
+
+        if BUILD in response_json:
+            # Set build
+            status_data[SEARCH_API][BUILD] = response_json[BUILD]
+
         if ELASTICSEARCH_CONNECTION in response_json:
             # Add the elasticsearch connection status
             status_data[SEARCH_API][ELASTICSEARCH_CONNECTION] = response_json[ELASTICSEARCH_CONNECTION]