Merge pull request #31 from ukgovdatascience/tidy-repository

Add hooks, add tests, improve functionality, refactor documentation, and add GitHub Actions

.flake8

@@ -5,4 +5,5 @@
 exclude = venv*,__pycache__,node_modules,bower_components,migrations
 ignore = D203,W503
 max-complexity = 9
-max-line-length = 120
+max-line-length = 88
+extend-ignore = E203

{{ cookiecutter.repo_name }}/.govcookiecutter/department_frameworks/GDS/request_template.md → .github/pull_request_template.md

@@ -1,30 +1,31 @@
 # Summary
 
-Add your summary here - keep it brief, to the point, and in plain English. For further information about pull requests,
-check out the [GDS Way][gds-way].
+Add your summary here - keep it brief, to the point, and in plain English. For further
+information about pull requests, check out the
+[GDS Way](https://gds-way.cloudapps.digital/standards/pull-requests.html).
 
 # Checklists
 
 <!--
 These are DO-CONFIRM checklists; it CONFIRMs that you have DOne each item.
 
-Outstanding actions should be completed before reviewers are assigned; if actions are irrelevant, please try and add a
-comment stating why.
+Outstanding actions should be completed before reviewers are assigned; if actions are
+irrelevant, please try and add a comment stating why.
 
-Incomplete pull/merge requests MAY be blocked until actions are resolved, or closed at the reviewers' discretion.
+Incomplete pull/merge requests MAY be blocked until actions are resolved, or closed at
+the reviewers' discretion.
 -->
 
 This pull/merge request meets the following requirements:
 
 - [ ] Code runs
 - [ ] Developments are **secure** and [**ethical**][data-ethics-framework]
 - [ ] You have made proportionate checks that the code works correctly
+- [ ] You have tested the build of the template
 - [ ] Test suite passes
-- [ ] Assumptions, and caveats log updated (see `docs/aqa/assumptions_caveats.md`), if necessary
 - [ ] [Minimum usable documentation][agilemodeling] written in the `docs` folder
 
 Comments have been added below around the incomplete checks.
 
 [agilemodeling]: http://agilemodeling.com/essays/documentLate.htm
 [data-ethics-framework]: https://www.gov.uk/government/publications/data-ethics-framework
-[gds-way]: https://gds-way.cloudapps.digital/standards/pull-requests.html

.github/workflows/govcookiecutter-build.yml

@@ -0,0 +1,63 @@
+name: govcookiecutter build
+
+on: [ push ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python: [ 3.6, 3.7, 3.8, 3.9 ]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install dependencies
+        run: |
+          if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "macOS" ]]; then
+            make requirements
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            pip install -U pip setuptools
+            pip install -r requirements.txt
+            pre-commit install
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi;
+        shell: bash
+      - name: Run pre-commit hooks
+        run: pre-commit run --all-files
+      - name: Create documentation
+        run: |
+          if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "macOS" ]]; then
+            sphinx-build -W -b html ./docs ./docs/_build
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            # TODO: Investigate why Sphinx build raises a warning on Windows but not Unix
+            sphinx-build -b html ./docs ./docs/_build
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi;
+        shell: bash
+      - name: Execute tests, and create coverage report
+        run: |
+          if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "macOS" ]]; then
+            make coverage_xml
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            coverage run -m pytest
+            coverage xml
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi;
+        shell: bash
+      - name: Upload coverage report to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          files: ./coverage.xml
+          env_vars: OS=${{ matrix.os }},PYTHON=${{ matrix.python }}

.github/workflows/govcookiecutter-template-build.yml

@@ -0,0 +1,74 @@
+name: govcookiecutter template build
+
+on: [ pull_request ]
+
+jobs:
+  template-build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # TODO: Fix R pre-commit hook issues with Windows
+        os: [ ubuntu-latest, macos-latest ]
+        python: [ 3.6, 3.7, 3.8, 3.9 ]
+        R:
+          - {using_R: No, version: N/A}
+          - {using_R: Yes, version: 4.0.4}
+          - {using_R: Yes, version: 4.0.5}
+          - {using_R: Yes, version: 4.1.0}
+
+    steps:
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install R ${{ matrix.R.version }}
+        uses: r-lib/actions/setup-r@v1
+        with:
+          r-version: ${{ matrix.R.version }}
+        if: ${{ matrix.R.using_r == 'Yes' }}
+      - name: Install other ${{ matrix.os }} R dependencies
+        run: sudo apt-get install libcurl4-openssl-dev
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.R.using_R == 'Yes' }}
+      - name: Install cookiecutter
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            # TODO: Add pip alternative, as Ubuntu 20.04LTS not currently supported (June 2021)
+            sudo apt-get install cookiecutter || pip install cookiecutter
+          elif [ "$RUNNER_OS" == "macOS" ]; then
+            brew install cookiecutter
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            pip install cookiecutter
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi;
+        shell: bash
+      - name: Create project from govcookiecutter for this commit
+        run: |
+          # Relies on the repository being public
+          cookiecutter ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git --no-input --checkout ${GITHUB_HEAD_REF} repo_name=example using_R=${{ matrix.R.using_R }}
+        shell: bash
+      - name: Install requirements, and pre-commit hooks, and check Sphinx build
+        run: |
+          cd example
+          git init
+          git add .
+          if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "macOS" ]]; then
+            make docs
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            pip install -U pip setuptools
+            pip install -r requirements.txt
+            pre-commit install
+            sphinx-build -b html ./docs ./docs/_build
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi;
+        shell: bash
+      - name: Run pre-commit hooks
+        run: |
+          cd example
+          pre-commit run --all-files
+          # TODO: Currently no tests - this gives exit code 5, which fails CI, so commenting out for now
+          # pytest

.pre-commit-config.yaml

@@ -1,28 +1,38 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
     hooks:
-    -   id: flake8
-        args: ["hooks", "tests", "{{ cookiecutter.repo_name }}/src"]
--   repo: https://github.com/Yelp/detect-secrets
+      - id: check-added-large-files
+        name: Check for files larger than 5 MB
+        args: [ "--maxkb=5120" ]
+      - id: end-of-file-fixer
+        name: Check for a blank line at the end of scripts (auto-fixes)
+      - id: trailing-whitespace
+        name: Check for trailing whitespaces (auto-fixes)
+  - repo: https://github.com/pycqa/isort
+    rev: 5.8.0
+    hooks:
+      - id: isort
+        name: isort - Sort Python imports (auto-fixes)
+        types: [ cython, pyi, python ]
+        args: [ "--profile", "black", "--filter-files" ]
+  - repo: https://github.com/psf/black
+    rev: 21.5b2 # Replace by any tag/version: https://github.com/psf/black/tags
+    hooks:
+      - id: black
+        name: black - consistent Python code formatting (auto-fixes)
+        language_version: python # Should be a command that runs python3.6+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        name: flake8 - Python linting
+  - repo: https://github.com/Yelp/detect-secrets
     rev: v1.0.3
     hooks:
-    -   id: detect-secrets
-        args: ["--baseline", ".secrets.baseline"]
+      - id: detect-secrets
+        name: detect-secrets - Detect secrets in staged code
+        args: [ "--baseline", ".secrets.baseline" ]
         exclude: .*/tests/.*
-- repo: https://github.com/kynan/nbstripout
-  rev: 0.3.9
-  hooks:
-    - id: nbstripout
-      args:
-        - --extra-keys
-        - "metadata.colab metadata.kernelspec cell.metadata.colab cell.metadata.executionInfo cell.metadata.id cell.metadata.outputId"
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
-    hooks:
-    -   id: check-added-large-files
-        args: ["--maxkb=5120"]
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace

.secrets.baseline

@@ -90,5 +90,5 @@
     }
   ],
   "results": {},
-  "generated_at": "2021-03-04T17:23:56Z"
+  "generated_at": "2021-06-14T10:43:14Z"
 }

CODE_OF_CONDUCT.md

@@ -1,11 +1,14 @@
 # Code of conduct for `govcookiecutter`
 
-Contributors to this repository hosted on `ukgovdatascience` are expected to follow the Contributor Covenant Code of
-Conduct, and those working within Her Majesty's Government are also expected to follow the Civil Service Code.
+Contributors to this repository hosted by `ukgovdatascience` are expected to follow the
+Contributor Covenant Code of Conduct, and those working within Her Majesty's Government
+are also expected to follow the Civil Service Code.
 
 ## Civil Service Code
 
-The Civil Service Code can be found [here][civil-service-code].
+Contributors working within Her Majesty's Government must review the
+[Civil Service Code][civil-service-code], and are expected to follow it in their
+contributions.
 
 ## Contributor Covenant Code of Conduct
 
@@ -15,19 +18,21 @@ Where this Code of Conduct says:
 
 - "Project", we mean this `govcookiecutter` GitHub repository;
 - "Maintainer", we mean the `ukgovdatascience` organisation owners; and
-- "Leadership", we mean both `ukgovdatascience` organisation owners, line managers, and other leadership within the
-  Government Digital Service.
+- "Leadership", we mean both `ukgovdatascience` organisation owners, line managers, and
+  other leadership within the Government Digital Service.
 
 ### Our Pledge
 
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make
-participation in our project, and our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education,
-socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+In the interest of fostering an open and welcoming environment, we as contributors and
+maintainers pledge to make participation in our project, and our community a
+harassment-free experience for everyone, regardless of age, body size, disability,
+ethnicity, sex characteristics, gender identity and expression, level of experience,
+education, socio-economic status, nationality, personal appearance, race, religion, or
+sexual identity and orientation.
 
 ### Our Standards
 
-Examples of behavior that contributes to creating a positive environment include:
+Examples of behaviour that contributes to creating a positive environment include:
 
 - Using welcoming and inclusive language
 - Being respectful of differing viewpoints and experiences
@@ -40,41 +45,51 @@ Examples of unacceptable behaviour by participants include:
 - The use of sexualised language or imagery and unwelcome sexual attention or advances
 - Trolling, insulting/derogatory comments, and personal or political attacks
 - Public or private harassment
-- Publishing others' private information, such as a physical or electronic address, without explicit permission
-- Other conduct which could reasonably be considered inappropriate in a professional setting
+- Publishing others' private information, such as a physical or electronic address,
+  without explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional
+  setting
 
 ### Our Responsibilities
 
-Project maintainers are responsible for clarifying the standards of acceptable behaviour and are expected to take
-appropriate and fair corrective action in response to any instances of unacceptable behavior.
+Project maintainers are responsible for clarifying the standards of acceptable
+behaviour and are expected to take appropriate and fair corrective action in response
+to any instances of unacceptable behaviour.
 
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits,
-issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any
-contributor for other behaviours that they deem inappropriate, threatening, offensive, or harmful.
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are not
+aligned to this Code of Conduct, or to ban temporarily or permanently any contributor
+for other behaviours that they deem inappropriate, threatening, offensive, or harmful.
 
 ### Scope
 
-This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the
-project or its community in public spaces. Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting as an appointed representative at an
-online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+This Code of Conduct applies within all project spaces, and it also applies when an
+individual is representing the project or its community in public spaces. Examples of
+representing a project or community include using an official project e-mail address,
+posting via an official social media account, or acting as an appointed representative
+at an online or offline event. Representation of a project may be further defined and
+clarified by project maintainers.
 
 ### Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at
-[[email protected]][email-address]. All complaints will be reviewed and investigated and
-will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated
-to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement
-policies may be posted separately.
+Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by
+contacting the project team at
+[[email protected]][email-address]. All complaints will be
+reviewed and investigated and will result in a response that is deemed necessary and
+appropriate to the circumstances. The project team is obligated to maintain
+confidentiality with regard to the reporter of an incident. Further details of
+specific enforcement policies may be posted separately.
 
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent
-repercussions as determined by other members of the project's leadership.
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may
+face temporary or permanent repercussions as determined by other members of the
+project's leadership.
 
 ### Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant], version 1.4, available at
-[https://www.contributor-covenant.org/version/1/4/code-of-conduct.html][contributor-covenant-code-of-conduct], and the
-`alphagov` Code of Conduct available at
+This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant],
+version 1.4, available at
+[https://www.contributor-covenant.org/version/1/4/code-of-conduct.html][contributor-covenant-code-of-conduct],
+and the `alphagov` Code of Conduct available at
 [https://github.com/alphagov/.github/blob/master/CODE_OF_CONDUCT.md][alphagov-code-of-conduct].
 
 [alphagov-code-of-conduct]: https://github.com/alphagov/.github/blob/master/CODE_OF_CONDUCT.md