Release 4.4.0 (#356)

askomics · Jul 1, 2022 · 399eb77 · 399eb77
1 parent f2658a9
commit 399eb77
Show file tree

Hide file tree

Showing 149 changed files with 5,825 additions and 1,555 deletions.
diff --git a/.github/workflows/lint_test.yml b/.github/workflows/lint_test.yml
@@ -8,7 +8,7 @@ jobs:
       uses: actions/checkout@v2
     - uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Install flake8
       run: pip install flake8
     - name: Flake8
@@ -33,7 +33,7 @@ jobs:
       uses: actions/checkout@v2
     - uses: actions/setup-python@v1
       with:
-        python-version: 3.6
+        python-version: 3.8
     - name: Update apt cache
       run: sudo apt-get update
     - name: Install python-ldap deps
@@ -50,7 +50,7 @@ jobs:
         docker pull xgaia/corese:latest
         docker pull xgaia/isql-api:2.1.1
         docker pull xgaia/simple-ldap:latest
-        docker run -d --name virtuoso -p 8891:8890 -p 1112:1111 -e DBA_PASSWORD=dba -e SPARQL_UPDATE=true -e DEFAULT_GRAPH=http://localhost:8891/DAV -t askomics/virtuoso:7.2.5.1 /bin/sh -c "netstat -nr | grep '^0\.0\.0\.0' | grep -oE '((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])' | grep -v '^0\.0\.0\.0' | sed 's/$/ askomics-host/' >> /etc/hosts && /virtuoso/virtuoso.sh"
+        docker run -d --name virtuoso -p 8891:8890 -p 1112:1111 -e DBA_PASSWORD=dba -e DEFAULT_GRAPH=http://localhost:8891/DAV -t askomics/virtuoso:7.2.5.1 /bin/sh -c "netstat -nr | grep '^0\.0\.0\.0' | grep -oE '((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])' | grep -v '^0\.0\.0\.0' | sed 's/$/ askomics-host/' >> /etc/hosts && /virtuoso/virtuoso.sh"
         sleep 1m
         docker run -d --name redis -p 6380:6379 -t redis:4.0
         docker run -d --name galaxy -p 8081:80 -t bgruening/galaxy-stable:20.05

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 This changelog was started for release 4.2.0.
 
+## [4.4.0] - 2022-07-01
+
+### Fixed
+
+- Fixed an issue with forms (missing field and entity name for label & uri fields) (Issue #255)
+- Fixed an issue with the data endpoint for FALDO entities (Issue #279)
+- Fixed an issue where integration would fail when setting a category type on a empty column (#334)
+- Fixed an issue with saved queries for non-logged users
+
+### Added
+
+- Added 'scaff' for autodetection of 'reference' columns
+- Added a 'Label' column type: only for second column in CSV files. Will use this value if present, else default to old behavior
+- Added button to hide FALDO relations (*included_in*)
+- Added 'target=_blank' in query results
+- Remote upload is now sent in a Celery task
+- Added 'Status' for files (for celery upload, and later for better file management)
+- Added tooltips to buttons in the query form (and other forms)
+- Added owl integration
+- Add better error management for RDF files
+- Added 'single tenant' mode: Send queries to all graphs to speed up
+- Added ontologies management
+- Added prefixes management
+- Added 'external graph' management for federated request: federated requests will only target this remote graph
+- Added support for multithread in web server, with the *WORKERS* env variable when calling make
+
+### Changed
+
+- Changed "Query builder" to "Form editor" in form editing interface
+- Changed abstraction building method for relations. (Please refer to #248 and #268). Correct 'phantom' relations
+- Changed abstraction building method for attributes. (Please refer to #321 and #324). Correct 'attributes' relations
+- Changed abstraction building method for 'strand': only add the required strand type, and not all three types (#277)
+- Updated documentation
+- Changed the sparql endpoint: now use the authenticated SPARQL endpoint instead of public endpoint. Write permissions are not required anymore
+- Reverted base docker image to alpine-13 to solve a docker issue
+
+### Removed
+
+- Removed "Remote endpoint" field for non-ttl file
+- Removed "Custom_uri" field for ttl file
+
+### Security
+
+- Bump axios from 0.21.1 to 0.21.2
+- Bump tar from 6.1.0 to 6.1.11
+- Bump @npmcli/git from 2.0.6 to 2.1.0
+- Bump path-parse from 1.0.6 to 1.0.7
+- Bump prismjs from 1.23.0 to 1.27.0
+- Bump simple-get from 2.8.1 to 2.8.2
+- Bump ssri from 6.0.1 to 6.0.2
+- Bump follow-redirects from 1.14.4 to 1.14.8
+- Bump mkdocs from 1.0.4 to 1.2.3 in /docs
+- Bump python-ldap from 3.3.1 to 3.4.0
+- Bump minimist from 1.2.5 to 1.2.6
 
 ## [4.3.1] - 2021-06-16
 

diff --git a/Makefile b/Makefile
@@ -9,6 +9,7 @@ FLASKOPTS=
 PYTESTOPTS=
 TESTFILE?=tests
 NTASKS?=1
+WORKERS?=1
 
 HOST?=0.0.0.0
 PORT?=5000
@@ -52,20 +53,20 @@ help:
 	@echo '    make clean                                                         Uninstall everything'
 	@echo '    make install [MODE=dev]                                            Install Python and Js dependencies (+ dev dependencies if MODE=dev)'
 	@echo '    make build [MODE=dev]                                              Build javascript (and watch for update if MODE=DEV)'
-	@echo '    make serve [MODE=dev] [HOST=0.0.0.0] [PORT=5000] [NTASKS=1]        Serve AskOmics at $(HOST):$(PORT)'
+	@echo '    make serve [MODE=dev] [HOST=0.0.0.0] [PORT=5000] [NTASKS=1] [WORKERS=1] Serve AskOmics at $(HOST):$(PORT)'
 	@echo '    make test                                                          Lint and test javascript and python code'
 	@echo '    make serve-doc [DOCPORT=8000]                                      Serve documentation at localhost:$(DOCPORT)'
 	@echo '    make update-base-url                                               Update all graphs from an old base_url to a new base_url'
 	@echo '    make clear-cache                                                   Clear abstraction cache'
 	@echo ''
 	@echo 'Examples:'
-	@echo '    make clean install build serve NTASKS=10                           Clean install and serve AskOmics in production mode, 10 celery tasks in parallel'
-	@echo '    make clean install serve MODE=dev NTASKS=10                        Clean install and serve AskOmics in development mode, 10 celery tasks in parallel'
+	@echo '    make clean install build serve NTASKS=10 WORKERS=2                 Clean install and serve AskOmics in production mode, 10 celery tasks in parallel, 2 workers on the web server'
+	@echo '    make clean install serve MODE=dev NTASKS=10 WORKERS=2              Clean install and serve AskOmics in development mode, 10 celery tasks in parallel, 2 workers on the web server'
 	@echo ''
 	@echo '    make clean install                                                 Clean install AskOmics'
 	@echo '    make clean install  MODE=dev                                       Clean install AskOmics in development mode'
-	@echo '    make serve NTASKS=10                                               Serve AskOmics, 10 celery tasks in parallel'
-	@echo '    make serve MODE=dev NTASKS=10                                      Serve AskOmics in development mode, 10 celery tasks in parallel'
+	@echo '    make serve NTASKS=10 WORKERS=2                                     Serve AskOmics, 10 celery tasks in parallel, 2 workers on the web server'
+	@echo '    make serve MODE=dev NTASKS=10 WORKERS=2                            Serve AskOmics in development mode, 10 celery tasks in parallel, 2 workers on the web server'
 	@echo ''
 	@echo '    make pytest MODE=dev TESTFILE=tests/test_api.py                    Test tests/test_api file only'
 
@@ -81,7 +82,7 @@ test-js: eslint
 
 eslint: check-node-modules
 	@echo -n 'Linting javascript...                                        '
-	$(NODEDIR)/.bin/eslint --config $(BASEDIR)/.eslintrc.yml "$(BASEDIR)/askomics/react/src/**"
+	$(NODEDIR)/.bin/eslint --config $(BASEDIR)/.eslintrc.yml "$(BASEDIR)/askomics/react/src/**" || { echo "ERROR"; exit 1; }
 	@echo "Done"
 
 test-python: pylint pytest
@@ -94,7 +95,7 @@ pytest: check-venv
 pylint: check-venv
 	@echo -n 'Linting python...                                            '
 	. $(ACTIVATE)
-	flake8 $(BASEDIR)/askomics $(BASEDIR)/tests --ignore=E501,W504
+	flake8 $(BASEDIR)/askomics $(BASEDIR)/tests --ignore=E501,W504 || { echo "ERROR"; exit 1; }
 	@echo "Done"
 
 serve: check-venv build-config create-user
@@ -106,7 +107,7 @@ serve-askomics: check-venv build-config create-user
 ifeq ($(MODE), dev)
 	FLASK_ENV=development FLASK_APP=app flask run --host=$(HOST) --port $(PORT)
 else
-	FLASK_ENV=production FLASK_APP=app gunicorn -b $(HOST):$(PORT) app
+	FLASK_ENV=production FLASK_APP=app gunicorn -w $(WORKERS) -b $(HOST):$(PORT) app
 endif
 
 serve-celery: check-venv build-config create-user
@@ -126,32 +127,32 @@ check-node-modules:
 
 build-config:
 	@echo -n 'Building config file...                                      '
-	bash cli/set_config.sh
+	bash cli/set_config.sh || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 
 create-user:
 	@echo -n 'Creating first user...                                       '
 	. $(ACTIVATE)
-	bash cli/set_user.sh
+	bash cli/set_user.sh || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 
 update-base-url: check-venv
 	@echo 'Updating base url...'
 	. $(ACTIVATE)
-	bash cli/update_base_url.sh
+	bash cli/update_base_url.sh || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 
 clear-cache: check-venv
 	@echo 'Clearing abstraction cache...'
 	. $(ACTIVATE)
-	bash cli/clear_cache.sh
+	bash cli/clear_cache.sh || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 
 build: build-js
 
 build-js: check-node-modules
 	@echo 'Building askomics.js...                                        '
-	$(NPM) run --silent $(NPMOPTS)
+	$(NPM) run --silent $(NPMOPTS) || { echo "ERROR"; exit 1; }
 	@echo '                                                             Done'
 
 install: install-python install-js
@@ -161,22 +162,22 @@ fast-install:
 
 install-python: check-python
 	@echo -n 'Building python virtual environment...                       '
-	$(PYTHON) -m venv $(VENVDIR)
+	$(PYTHON) -m venv $(VENVDIR) || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 	@echo -n 'Sourcing Python virtual environment...                       '
-	. $(ACTIVATE)
+	. $(ACTIVATE) || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 	@echo -n 'Upgrading pip...                                             '
-	$(PIP) install --upgrade pip > /dev/null
+	$(PIP) install --upgrade pip > /dev/null || { echo "ERROR"; exit 1; }
 	@echo 'Done'
 	@echo 'Installing Python dependencies inside virtual environment... '
-	$(PIP) install -e . > /dev/null
-	PIPENV_VERBOSITY=-1 $(PIPENV) install $(PIPENVOPTS)
+	$(PIP) install -e . > /dev/null || { echo "ERROR"; exit 1; }
+	PIPENV_VERBOSITY=-1 $(PIPENV) install $(PIPENVOPTS) || { echo "ERROR"; exit 1; }
 	@echo '                                                             Done'
 
 install-js: check-npm
 	@echo  'Installing javascript dependencies inside node_modules...    '
-	$(NPM) install --silent
+	$(NPM) install || { echo "ERROR"; exit 1; }
 	@echo '                                                             Done'
 
 clean: clean-js clean-python

diff --git a/Pipfile b/Pipfile
@@ -5,15 +5,15 @@ name = "pypi"
 
 [packages]
 werkzeug = "==0.16.1"
-flask = "==1.1.4"
+flask = "<2"
 flask-reverse-proxy-fix = "*"
 validate-email = "*"
 gunicorn = "*"
 python-magic = "*"
 rdflib = "*"
 sparqlwrapper = "*"
 requests = "*"
-celery = "*"
+celery = "==5.0.5"
 redis = "*"
 watchdog = "*"
 gitpython = "*"