Merge pull request #245 from emmo-repo/cwa/close-243-223-pre-commit-l…

…inting-formatting

Implement `pre-commit`.
Pre-commit is a tool that one installs in the git repository
and runs prior to committing, checking various parts of the
changed files.
For more information go to
[pre-commit.com](https://pre-commit.com).

Several hooks from the pre-commit-hooks repository
maintained by pre-commit themselves are added:

- check-symlinks
- check-xml (running for all xml, rdf, and ttl files)
- check-yaml
- destroyed-symlinks
- end-of-file-fixer
- requirements-txt-fixer
- trailing-whitespace

To understand what each of these hooks do, go the
[README for the pre-commit-hooks repo](https://github.com/pre-commit/pre-commit-hooks#readme).

`bandit` and `safety` is introduced.
`bandit` will be introduced both as a `pre-commit` hook,
as well as running in the CI.
`safety` will only run in the CI.

`pylint` is introduced and will take over from `flake8`.
As `pylint` cannot run properly in pre-commit unless it's
running in the local environment, it is used as a local hook
and added to the list of dependencies.

A new install extra (and list of requirements) is added, containing
`pre-commit`, `pylint`, and `pytest`.
These should all be using the latest version, so they have been
specified using `~=`.

Finally, `black` has been added as the chosen Python style and formatter.
The maximum line-length has been kept at 80 characters, but otherwise
it does its stuff.
The main changes in this merge is due to these changes.

A hook for MyPy has been added to the configuration file, but as yet
commented out as this should be added together with a full type
annotation of the code.

.github/workflows/ci_workflow.yml

@@ -9,71 +9,121 @@ on:
 
 jobs:
 
-  lint:
-    name: Lint with flake8
+  pre-commit:
+    name: pre-commit
     runs-on: ubuntu-latest
+
     steps:
-    - uses: actions/[email protected]
+    - name: Checkout repository
+      uses: actions/checkout@v2
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v2
       with:
         python-version: 3.7
 
-    - name: Lint with flake8
+    - name: Install dependencies
       run: |
-        pip install flake8
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude "tests/*"
-        flake8 . --count --statistics --extend-exclude "tests/*"
+        python -m pip install --upgrade pip
+        pip install -U setuptools wheel
+
+        while IFS="" read -r line || [ -n "${line}" ]; do
+          if [[ "${line}" =~ ^pre-commit.*$ ]]; then
+            pre_commit="${line}"
+          fi
+        done < requirements_dev.txt
+
+        while IFS="" read -r line || [ -n "${line}" ]; do
+          if [[ "${line}" =~ ^invoke.*$ ]]; then
+            invoke="${line}"
+          fi
+        done < requirements_docs.txt
+
+        pip install ${pre_commit} ${invoke}
+
+    - name: Run pre-commit
+      run: SKIP=pylint pre-commit run --all-files --show-diff-on-failure
+
+
+  pylint-safety:
+    name: PyLint and safety
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U setuptools wheel
+
+        pip install -U -r requirements.txt -r requirements_dev.txt -r requirements_docs.txt
+        pip install safety
+
+    - name: Run PyLint
+      run: pylint --rcfile=pyproject.toml *.py tools
+
+    - name: Run safety
+      run: pip freeze | safety check --stdin
+
 
   tests:
-    name: Pytest and demos
+    name: pytest
     runs-on: ubuntu-latest
+
     steps:
-    - uses: actions/[email protected]
+    - name: Checkout repository
+      uses: actions/checkout@v2
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v2
       with:
         python-version: 3.7
 
-    - name: Install other dependencies
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y graphviz openjdk-11-jre-headless
+
+    - name: Install Python dependencies (and EMMOntoPy)
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install pytest
-        sudo apt-get install -y graphviz openjdk-11-jre-headless
+        pip install -U setuptools wheel
 
-    - name: Install EMMOntoPy
-      run: python setup.py install
+        pip install -U -e .[dev]
 
     - name: Test with pytest
       run: pytest
 
-    - name: Run vertical demo
-      run: |
-        cd demo/vertical
-        #python define_ontology.py
-        #python plot_ontology.py
-        cd -
-    - name: Run horizontal demo
-      run: |
-        cd demo/horizontal
-        #python step1_generate_metadata.py
-        #python step2_define_metadata.py
-        #python step3_instantiate.py
-        #python step4_map_instance.py
-        cd -
+    # - name: Run vertical demo
+    #   run: |
+    #     cd demo/vertical
+    #     python define_ontology.py
+    #     python plot_ontology.py
+    #     cd -
+
+    # - name: Run horizontal demo
+    #   run: |
+    #     cd demo/horizontal
+    #     python step1_generate_metadata.py
+    #     python step2_define_metadata.py
+    #     python step3_instantiate.py
+    #     python step4_map_instance.py
+    #     cd -
+
 
   ontodoc:
     name: EMMO documentation (test using ontodoc)
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v2
       with:
         python-version: 3.7
 
@@ -117,6 +167,7 @@ jobs:
         python ../../tools/ontodoc -t emmo.md emmo-inferred emmo.pdf
         cd -
 
+
   build_package:
     name: Build Python package and install (integrity test)
     runs-on: ubuntu-latest
@@ -126,10 +177,10 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2.3.4
+      uses: actions/checkout@v2
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.2.2
+      uses: actions/setup-python@v2
       with:
         python-version: 3.7
 

.pre-commit-config.yaml

@@ -0,0 +1,62 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+  - id: check-symlinks
+  - id: check-xml
+    name: Check XML
+    files: \.(xml|rdf|ttl)$
+  - id: check-yaml
+    name: Check YAML
+  - id: destroyed-symlinks
+  - id: end-of-file-fixer
+  - id: requirements-txt-fixer
+    name: Fix requirements*.txt
+    files: ^requirements.*\.txt$
+  - id: trailing-whitespace
+    args: [--markdown-linebreak-ext=md]
+
+- repo: https://github.com/ambv/black
+  rev: 21.10b0
+  hooks:
+  - id: black
+    name: Blacken
+
+- repo: https://github.com/PyCQA/bandit
+  rev: '1.7.0'
+  hooks:
+  - id: bandit
+    args: [-r]
+    exclude: ^tests/.*$
+
+# - repo: https://github.com/pre-commit/mirrors-mypy
+#   rev: v0.910
+#   hooks:
+#   - id: mypy
+#     exclude: ^tests/.*$
+
+- repo: local
+  hooks:
+  - id: update-docs-api-reference
+    name: Update API Reference in Documentation
+    entry: invoke
+    args: [create-api-reference-docs, --pre-clean, --pre-commit]
+    language: python
+    pass_filenames: false
+    files: ^(emmopy|ontopy)/.*\.py$
+    description: Update the API Reference documentation, whenever a Python file is touched in the code base.
+  - id: update-docs-index
+    name: Update Landing Page for Documentation
+    entry: invoke
+    args: [create-docs-index]
+    language: python
+    pass_filenames: false
+    files: ^README.md$
+    description: Update the landing page for the documentation if the source file (README.md) is changed.
+  - id: pylint
+    name: pylint
+    entry: pylint
+    language: python
+    types: [python]
+    require_serial: true
+    exclude: ^tests/.*$