resolve conflict

.github/workflows/main.yaml

@@ -46,4 +46,24 @@ jobs:
 
       - name: Run pytest
         run: |
-          pytest -v
+          pytest -v
+
+  commit-hooks:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.6.8
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pre-commit install
+
+      - name: Check commit hooks
+        run: |
+          pre-commit run --all-files

.pre-commit-config.yaml

@@ -30,6 +30,15 @@ repos:
         language: script
         stages: [commit]
 
+#works
+-     repo: local
+      hooks:
+      - id: mixed-line-endings
+        entry: pre-commits/mixed_line_endings.py
+        name: Check for consistent end of line type LF to CRLF to CR (auto-fixes)
+        language: script
+        stages: [commit]
+
 #works
 #if using on different file types, it will need a seperate hook per file type
 -   repo: local
@@ -76,18 +85,17 @@ repos:
         stages: [commit]
 
 
-# #needs to remove the password in hello_world.py
-# -   repo: local
-#     hooks:
-#       - id: detect-secrets
-#         entry: detect-secrets
-#         name: detect-secrets - Detect secrets in staged code
-#         #args: [ "--baseline", ".secrets.baseline" ]
-#         args: [scan, audit]
-#         language: system
-#         types: [python]
-#         stages: [commit]
-#         exclude: .*/tests/.*|^\.cruft\.json$
+# works in testing
+-   repo: local
+    hooks:
+      - id: detect-secrets
+        entry: detect-secrets-hook
+        name: detect-secrets - Detect secrets in staged code
+        args: [ "--baseline", ".secrets.baseline" ]
+        #args: [scan, audit]
+        language: system
+        types: [python]
+        stages: [commit]
 
 
 

docs/contributor_guide/CONTRIBUTING.md

@@ -36,14 +36,14 @@ documentation][docs-pre-commit-hooks].
 
 ## Code conventions
 
-Code written for this project should follow [PEP 8 coding conventions](pep8), [project naming conventions](docs-naming) and the guidance on [quality assurance of code for analysis and research](duck-book) (also known as the Duck Book).
+Code written for this project should follow [PEP 8 coding conventions][pep8], [project naming conventions][docs-naming] and the guidance on [quality assurance of code for analysis and research][duck-book] (also known as the Duck Book).
 
 ### Git and GitHub
 
 We use Git to version control the source code and out source code is stored on
 GitHub.
 
-We follow the [GitHub flow](github-flow) workflow. This means that we create
+We follow the [GitHub flow][github-flow] workflow. This means that we create
 feature branches of the `main` branch and merge them back to `main` once they
 meet the definition of done. We give our branches short but informative names,
 in lowercase and separated with hypens. Where applicable, we start branch names
@@ -53,16 +53,20 @@ with the respective Jira ticket number. For example,
 We commit regularly, with distinct chunks of work where possible. We write
 short but informative commit messages, starting with a capitalised
 present-tense verb, for example `Add`, `Fix`. When pair-programming, we
-[add co-authors to the commit](git-coauthor). We add
-[longer commit messages](long-commit) for larger or more complex commits, for
+[add co-authors to the commit][git-coauthor]. We add
+[longer commit messages][long-commit] for larger or more complex commits, for
 example (squash) merge commits.
 
 We open a pull request to `main` once we have working code that meets a user
 need, for example meets the definition of done on the Jira ticket. Pull
 requests must be reviewed by at least one member of the team before merging.
-Reviews should follow the [pull request template](pr-template). If we want review on code that does not yet meet the definition of done, we open a draft
+Reviews should follow the [pull request template][pr-template]. If we want review on code that does not yet meet the definition of done, we open a draft
 pull request. Once a branch has been reviewed, it can be merged. We prefer to use squash merges, in order to simplify the `main` branch commit history. After merging the feature branch should be deleted.
 
+### Functions
+
+We prefer writing functions over classes to make it easier for beginners to understand the code. [Type hints][typing] should be used when writing functions. We prefer functions to return `pandas.DataFrame` rather than `pandas.Series`, for example when deriving new (temporary) variables.
+
 ### Markdown
 
 Local links can be written as normal, but external links should be referenced at the
@@ -83,6 +87,10 @@ tests, enter the following command in your terminal:
 ```shell
 pytest
 ```
+Our testing approach is:
+- use `.csv` files containing simple minimal input and output data for a function to be tested
+- individual test cases should be separated into different `.csv` files and grouped into folders
+- the name of the test data `.csv` files should reflect the test case and the folder name should be the same as the module/function
 
 ### Code coverage
 
@@ -139,3 +147,4 @@ build the documentation into an accessible, searchable website.
 [github-flow]: https://docs.github.com/en/get-started/using-github/github-flow
 [git-coauthor]: https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/creating-a-commit-with-multiple-authors
 [long-commit]: https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
+[typing]: https://docs.python.org/3/library/typing.html

pre-commits/check_added_large_files.py

@@ -4,24 +4,20 @@
 import json
 import math
 import os
-from typing import Optional
-from typing import Sequence
-from typing import Set
+from typing import Optional, Sequence, Set
 
-from pre_commit_hooks.util import added_files
-from pre_commit_hooks.util import CalledProcessError
-from pre_commit_hooks.util import cmd_output
+from pre_commit_hooks.util import CalledProcessError, added_files, cmd_output
 
 
 def _lfs_files() -> Set[str]:
     """Private function."""
     try:
         # Introduced in git-lfs 2.2.0, first working in 2.2.1
-        lfs_ret = cmd_output('git', 'lfs', 'status', '--json')
+        lfs_ret = cmd_output("git", "lfs", "status", "--json")
     except CalledProcessError:  # pragma: no cover (with git-lfs)
         lfs_ret = '{"files":{}}'
 
-    return set(json.loads(lfs_ret)['files'])
+    return set(json.loads(lfs_ret)["files"])
 
 
 def _find_large_added_files(filenames: Sequence[str], maxkb: int) -> int:
@@ -32,7 +28,7 @@ def _find_large_added_files(filenames: Sequence[str], maxkb: int) -> int:
     for filename in (added_files() & set(filenames)) - _lfs_files():
         kb = int(math.ceil(os.stat(filename).st_size / 1024))
         if kb > maxkb:
-            print(f'{filename} ({kb} KB) exceeds {maxkb} KB.')
+            print(f"{filename} ({kb} KB) exceeds {maxkb} KB.")
             retv = 1
 
     return retv
@@ -42,17 +38,20 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     """Entry function for script."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        'filenames', nargs='*',
-        help='Filenames pre-commit believes are changed.',
+        "filenames",
+        nargs="*",
+        help="Filenames pre-commit believes are changed.",
     )
     parser.add_argument(
-        '--maxkb', type=int, default=500,
-        help='Maxmimum allowable KB for added files',
+        "--maxkb",
+        type=int,
+        default=500,
+        help="Maxmimum allowable KB for added files",
     )
 
     args = parser.parse_args(argv)
     return _find_large_added_files(args.filenames, args.maxkb)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     exit(main())

pre-commits/check_merge_conflict.py

@@ -2,54 +2,49 @@
 """Pre commit hook to check for merge conflict flags in file."""
 import argparse
 import os.path
-from typing import Optional
-from typing import Sequence
-
+from typing import Optional, Sequence
 
 CONFLICT_PATTERNS = [
-    b'<<<<<<< ',
-    b'======= ',
-    b'=======\n',
-    b'>>>>>>> ',
+    b"<<<<<<< ",
+    b"======= ",
+    b"=======\n",
+    b">>>>>>> ",
 ]
 
 
 def _is_in_merge() -> int:
     """Private function."""
-    return (
-        os.path.exists(os.path.join('.git', 'MERGE_MSG')) and
-        (
-            os.path.exists(os.path.join('.git', 'MERGE_HEAD')) or
-            os.path.exists(os.path.join('.git', 'rebase-apply')) or
-            os.path.exists(os.path.join('.git', 'rebase-merge'))
-        )
+    return os.path.exists(os.path.join(".git", "MERGE_MSG")) and (
+        os.path.exists(os.path.join(".git", "MERGE_HEAD"))
+        or os.path.exists(os.path.join(".git", "rebase-apply"))
+        or os.path.exists(os.path.join(".git", "rebase-merge"))
     )
 
 
 def main(argv: Optional[Sequence[str]] = None) -> int:
     """Entry function for script."""
     parser = argparse.ArgumentParser()
-    parser.add_argument('filenames', nargs='*')
-    parser.add_argument('--assume-in-merge', action='store_true')
+    parser.add_argument("filenames", nargs="*")
+    parser.add_argument("--assume-in-merge", action="store_true")
     args = parser.parse_args(argv)
 
     if not _is_in_merge() and not args.assume_in_merge:
         return 0
 
     retcode = 0
     for filename in args.filenames:
-        with open(filename, 'rb') as inputfile:
+        with open(filename, "rb") as inputfile:
             for i, line in enumerate(inputfile):
                 for pattern in CONFLICT_PATTERNS:
                     if line.startswith(pattern):
                         print(
                             f'Merge conflict string "{pattern.decode()}" '
-                            f'found in {filename}:{i + 1}',
+                            f"found in {filename}:{i + 1}",
                         )
                         retcode = 1
 
     return retcode
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     exit(main())

pre-commits/end_of_line_fixer.py

@@ -2,9 +2,7 @@
 """Pre commit hook to ensure single blank line at end of python file."""
 import argparse
 import os
-from typing import IO
-from typing import Optional
-from typing import Sequence
+from typing import IO, Optional, Sequence
 
 
 def _fix_file(file_obj: IO[bytes]) -> int:
@@ -17,13 +15,13 @@ def _fix_file(file_obj: IO[bytes]) -> int:
         return 0
     last_character = file_obj.read(1)
     # last_character will be '' for an empty file
-    if last_character not in {b'\n', b'\r'} and last_character != b'':
+    if last_character not in {b"\n", b"\r"} and last_character != b"":
         # Needs this seek for windows, otherwise IOError
         file_obj.seek(0, os.SEEK_END)
-        file_obj.write(b'\n')
+        file_obj.write(b"\n")
         return 1
 
-    while last_character in {b'\n', b'\r'}:
+    while last_character in {b"\n", b"\r"}:
         # Deal with the beginning of the file
         if file_obj.tell() == 1:
             # If we've reached the beginning of the file and it is all
@@ -40,7 +38,7 @@ def _fix_file(file_obj: IO[bytes]) -> int:
     # newlines.  If we find extraneous newlines, then backtrack and trim them.
     position = file_obj.tell()
     remaining = file_obj.read()
-    for sequence in (b'\n', b'\r\n', b'\r'):
+    for sequence in (b"\n", b"\r\n", b"\r"):
         if remaining == sequence:
             return 0
         elif remaining.startswith(sequence):
@@ -54,21 +52,21 @@ def _fix_file(file_obj: IO[bytes]) -> int:
 def main(argv: Optional[Sequence[str]] = None) -> int:
     """Entry function for script."""
     parser = argparse.ArgumentParser()
-    parser.add_argument('filenames', nargs='*', help='Filenames to fix')
+    parser.add_argument("filenames", nargs="*", help="Filenames to fix")
     args = parser.parse_args(argv)
 
     retv = 0
 
     for filename in args.filenames:
         # Read as binary so we can read byte-by-byte
-        with open(filename, 'rb+') as file_obj:
+        with open(filename, "rb+") as file_obj:
             ret_for_file = _fix_file(file_obj)
             if ret_for_file:
-                print(f'Fixing {filename}')
+                print(f"Fixing {filename}")
             retv |= ret_for_file
 
     return retv
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     exit(main())