Merge pull request #10 from FallenChromium/dockerized-web

Docker support

.dockerignore

@@ -0,0 +1,7 @@
+.git
+.gitignore
+.github
+Dockerfile
+.dockerignore
+node_modules
+docs

Dockerfile

@@ -0,0 +1,38 @@
+#nodejs stage to build frontent 
+FROM node:16-alpine AS web-buildenv
+
+WORKDIR /sc-web   
+#Install sc-web build-time dependencies
+COPY package.json package-lock.json ./
+RUN npm install
+#Build sc-web
+COPY . .
+RUN npm run build
+
+FROM ubuntu:focal as runtime 
+USER root
+
+WORKDIR /sc-web
+# dependencies
+COPY --from=web-buildenv /sc-web/requirements.txt /sc-web/requirements.txt
+
+# install runtime dependencies
+# tini is a lightweight PID1 to forward interrupt signals
+RUN apt update && apt --no-install-recommends -y install python3-pip python3 tini
+RUN python3 -m pip install --no-cache-dir --default-timeout=100 -r requirements.txt
+
+# the app itself
+COPY --from=web-buildenv /sc-web/client /sc-web/client
+COPY --from=web-buildenv /sc-web/server /sc-web/server
+
+# Needed to populate KB on startup
+COPY --from=web-buildenv /sc-web/components /sc-web/components
+COPY --from=web-buildenv /sc-web/repo.path /sc-web/repo.path
+
+# config
+COPY --from=web-buildenv /sc-web/sc-web.ini /sc-web/sc-web.ini
+
+WORKDIR /sc-web/server
+
+EXPOSE 8000
+ENTRYPOINT ["tini", "--", "/usr/bin/python3", "/sc-web/server/app.py" ]

README.md

@@ -1,7 +1,19 @@
-## About
+# About
 Current repository contains web interface for ostis platform ([here](https://github.com/ostis-ai) you can find all ostis modules).
 
-## Installation
+# Installation
+## Using Docker
+Running image:
+```sh
+docker run --rm -it -p 8000:8000 ostis/sc-web --server_host=... <any other arguments of app.py>
+```
+### Connecting to sc-server 
+You will need to pass `--server_host=<your hostname>` flag to the container. 
+- In case the `sc-server` is running on another machine or container, simply put its hostname in the template. 
+- If you're running `sc-machine` locally, use [this guide](https://www.howtogeek.com/devops/how-to-connect-to-localhost-within-a-docker-container/) to set up connection to your host's `localhost`. Writing `--server_host=localhost` **will not work**. 
+
+In non-production environment it's easier to pass `--network=host` flag to `docker run` command to connect to `sc-server` running locally.
+## Installing manually
 ### Installing requirements:
 * python3
 * pip
@@ -12,7 +24,7 @@ Current repository contains web interface for ostis platform ([here](https://git
 
 You can install deps with
 
-```shell
+```sh
     pip3 install -r requirements.txt
     cd ./scripts
     ./install_deps_ubuntu.sh    #for debian based distros
@@ -21,7 +33,7 @@ You can install deps with
 ### Building
 To install current module you may use internal scripts. Run
 
-```shell
+```sh
     cd scripts
     ./install.sh
 ```

repo.path

@@ -0,0 +1 @@
+./components/scg/kb

requirements.txt

@@ -2,4 +2,4 @@ tornado
 sqlalchemy
 numpy
 configparser
-py-sc-client==0.2.0
+py-sc-client==0.2.2

scripts/run_scweb.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
+APP_ROOT_PATH=$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd)
 
-cd ../server/
-python3 app.py
-cd -
+python3 "$APP_ROOT_PATH"/server/app.py "$@"

server/app.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import configparser
+from glob import glob
 import logging
 import signal
 from abc import ABC
@@ -22,8 +23,13 @@
 from handlers.nl import NaturalLanguageSearch
 from keynodes import KeynodeSysIdentifiers
 
+from os.path import join, abspath, relpath, commonprefix, isdir, isfile, exists, dirname, splitext
+import re
+
 is_closing = False
 
+REPO_FILE_EXT = ".path"
+REPO_FILE_PATH = join(dirname(abspath(__file__)), "../repo.path")
 
 def signal_handler(signum, frame):
     global is_closing
@@ -53,8 +59,8 @@ def set_extra_headers(self, path):
 def main():
     logging.getLogger('asyncio').setLevel(logging.WARNING)
 
-    tornado.options.define("static_path", default="../client/static", help="path to static files directory", type=str)
-    tornado.options.define("templates_path", default="../client/templates", help="path to template files directory",
+    tornado.options.define("static_path", default=join(dirname(abspath(__file__)), "../client/static"), help="path to static files directory", type=str)
+    tornado.options.define("templates_path", default=join(dirname(abspath(__file__)),"../client/templates"), help="path to template files directory",
                            type=str)
     tornado.options.define("event_wait_timeout", default=10, help="time to wait commands processing", type=int)
     tornado.options.define("idtf_search_limit", default=100,
@@ -99,6 +105,12 @@ def main():
     # prepare logger
     logger_sc.init()
 
+    # load scs required for sc-web server
+    ingestion_results = client.create_elements_by_scs(read_scs_fragments(REPO_FILE_PATH))
+    if False in ingestion_results:
+        logger.error("Error uploading scs files to the sc-server. Exiting...")
+        exit(1)
+
     ScKeynodes().resolve_identifiers([KeynodeSysIdentifiers])
 
     rules = [
@@ -160,6 +172,75 @@ def main():
     logger.info("Close connection with sc-server")
     client.disconnect()
 
+scs_paths = set()
+scs_exclude_paths = set()
+def search_kb_sources(root_path: str):
+
+    if not exists(root_path):
+        print(root_path, "does not exist.")
+        exit(1)
+
+    elif splitext(root_path)[1] == REPO_FILE_EXT:
+        print()
+        with open(join(root_path), 'r') as root_file:
+            for line in root_file.readlines():
+                # ignore comments and empty lines
+                line = line.replace('\n', '')
+                # note: with current implementation, line is considered a comment if it's the first character
+                # in the line
+                if line.startswith('#') or re.match(r"^\s*$", line):
+                    continue
+                elif line.startswith('!'):
+                    absolute_path = abspath(join(dirname(root_path), line[1:]))
+                    scs_exclude_paths.add(absolute_path)
+                else:
+                    absolute_path = abspath(join(dirname(root_path), line))
+                    # ignore paths we've already checked
+                    if absolute_path not in scs_paths:
+                        # recursively check each repo entry
+                        search_kb_sources(absolute_path)
+
+    else:
+        scs_paths.add(root_path)
+
+    return(scs_paths, scs_exclude_paths)
+
+# read scs fragments unless they are excluded by repo.path
+def read_scs_fragments(root_path: str):
+    scs_fragments = []
+    paths, exclude_paths = search_kb_sources(root_path)
+    for path in paths:
+        # search for all scs in all subfolders of a path
+        if isdir(path):
+            for filename in glob(path + '/**/*.scs', recursive=True):
+                excluded = False
+
+                # check if it's inside excluded paths
+                for path in exclude_paths:
+                    if commonprefix([filename, path]) == path:
+                        excluded = True
+
+                if excluded == True:
+                    print("Excluded")
+                    continue
+                else:
+                    with open(filename, 'r') as f:
+                        scs_fragments.append(f.read())
+
+        elif isfile(path) and splitext(path)[1] == '.scs':
+            if path not in exclude_paths: 
+                with open(path, 'r') as f:
+                        scs_fragments.append(f.read())
+
+        elif not exists(path):
+            print("Error: path '%s' does not exist" % path)
+            exit(1)
+
+        else:
+            continue
+
+    return scs_fragments
+
 
 if __name__ == "__main__":
     main()

server/logger_sc.py

@@ -11,7 +11,7 @@
 
 def init():
     config = configparser.ConfigParser()
-    config.read(os.path.join(os.getcwd(), "../sc-web.ini"))
+    config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../sc-web.ini"))
 
     group = config["sc-web"]