diff --git a/.coveragerc b/.coveragerc index bd9f2ce..8888120 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,12 @@ +# .coveragerc to control coverage.py +# https://coverage.readthedocs.io/en/latest/config.html +# https://github.com/nedbat/coveragepy/blob/master/doc/config.rst [run] +data_file = ./build/.coverage source = python_boilerplate omit = python_boilerplate/__main__.py + python_boilerplate/__init__.py + tests/* [report] exclude_lines = @@ -18,3 +24,6 @@ exclude_lines = # Don't complain if non-runnable code isn't run: if 0: if __name__ == .__main__.: + +[html] +directory = ./build/htmlcov diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8a26e7a..553c941 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,14 +10,12 @@ on: - '**.md' - '_config.yml' - '**.tweet' - tags: "*" jobs: compilation: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Setup Python uses: actions/setup-python@v2 with: @@ -25,20 +23,20 @@ jobs: architecture: x64 # https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#caching-packages cache: "pipenv" - - name: Install Dependencies with pipenv run: | pip install pipenv pipenv install --deploy --dev - - run: pipenv run isort --recursive --diff . - run: pipenv run black --check . - run: pipenv run flake8 - run: pipenv run mypy - - name: Python tests with pytest - run: pipenv run pytest --cov --cov-fail-under=70 --capture=no --log-cli-level=INFO - - name: Build docker image - run: docker build . -t python_boilerplate:smoke-test-tag - - name: Smoke test docker image + - name: Python Tests with pytest + run: pipenv run pytest --cov --cov-fail-under=85 --capture=no --log-cli-level=INFO -n auto + - name: Build Docker Image + run: | + docker build . -t python_boilerplate:smoke-test-tag + docker inspect python_boilerplate:smoke-test-tag + - name: Smoke Test Docker Image run: | docker run --rm python_boilerplate:smoke-test-tag param_3_from_command_line diff --git a/.gitignore b/.gitignore index 7acb828..82cbf47 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.gz logs/ data/ +build/ ### PyCharm ### .idea @@ -67,6 +68,7 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ +.pytest_report/ .tox/ .nox/ .coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe1786a..6923878 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,53 +1,61 @@ # See https://pre-commit.com/ for usage and config repos: -- repo: local - hooks: - - id: isort - name: isort - stages: [commit] - language: system - entry: pipenv run isort - types: [python] - - - id: black - name: black - stages: [commit] - language: system - entry: pipenv run black - types: [python] - - - id: flake8 - name: flake8 - stages: [commit] - language: system - entry: pipenv run flake8 - types: [python] - exclude: setup.py - - - id: mypy - name: mypy - stages: [commit] - language: system - entry: pipenv run mypy - types: [python] - require_serial: true - - - id: pytest - name: pytest - stages: [commit] - language: system - # https://github.com/pytest-dev/pytest/issues/5502#issuecomment-1020761655 - # Prevent Pytest logging error like: ValueError: I/O operation on closed file. - entry: pipenv run pytest --cov --cov-report html --capture=no --log-cli-level=DEBUG - types: [python] - pass_filenames: false - - - id: pytest-cov - name: pytest - stages: [push] - language: system - # https://github.com/pytest-dev/pytest/issues/5502#issuecomment-1020761655 - # Prevent Pytest logging error like: ValueError: I/O operation on closed file. - entry: pipenv run pytest --cov --cov-fail-under=85 --capture=no --log-cli-level=INFO - types: [python] - pass_filenames: false + # https://github.com/pre-commit/pre-commit-hooks/releases/tag/v4.3.0 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: mixed-line-ending + args: [ --fix, lf ] + - id: check-yaml + - id: debug-statements + - id: name-tests-test + args: [ --pytest-test-first ] + - id: detect-private-key + - repo: local + hooks: + - id: isort + name: isort + stages: [ commit ] + language: system + entry: pipenv run isort + types: [ python ] + - id: black + name: black + stages: [ commit ] + language: system + entry: pipenv run black + types: [ python ] + - id: flake8 + name: flake8 + stages: [ commit ] + language: system + entry: pipenv run flake8 + types: [ python ] + exclude: setup.py + - id: mypy + name: mypy + stages: [ commit ] + language: system + entry: pipenv run mypy + types: [ python ] + require_serial: true + - id: pytest + name: pytest + stages: [ commit ] + language: system + # https://github.com/pytest-dev/pytest/issues/5502#issuecomment-1020761655 + # Prevent Pytest logging error like: ValueError: I/O operation on closed file. + entry: pipenv run pytest --cov --cov-report html --html=./build/.pytest_report/report.html --self-contained-html --log-cli-level=DEBUG -n auto + types: [ python ] + pass_filenames: false + - id: pytest-cov + name: pytest + stages: [ push ] + language: system + # https://github.com/pytest-dev/pytest/issues/5502#issuecomment-1020761655 + # Prevent Pytest logging error like: ValueError: I/O operation on closed file. + entry: pipenv run pytest --cov --cov-fail-under=85 --capture=no --log-cli-level=INFO -n auto + types: [ python ] + pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d97d67..2182be0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +# [4.0.0](https://github.com/johnnymillergh/muscle-and-fitness-server/compare/3.0.0...4.0.0) (2022-09-18) + + +### Features + +* **$async:** support decorate function with `@async_function` ([44a0a7e](https://github.com/johnnymillergh/python_boilerplate/commit/44a0a7eaf5ecf68ff0b8e170c0cb7c9c836b1fa9)) +* **$flake8:** add more code constraints ([b978b9b](https://github.com/johnnymillergh/python_boilerplate/commit/b978b9b0d43dfb84ccc9e48b5ed51094baf531a6)) +* **$GitHooks:** add more hooks for pre-commit ([5114647](https://github.com/johnnymillergh/python_boilerplate/commit/5114647115151d74f68f3855281487e753899215)) +* **$loguru:** display process id in log ([5a0177b](https://github.com/johnnymillergh/python_boilerplate/commit/5a0177b6517ea64e296d855057e35f91c028d4ab)) +* **$pandas:** add usage example of using pandas DataFrame to generate CSV ([161c7ba](https://github.com/johnnymillergh/python_boilerplate/commit/161c7bae5f0c95fc9e4d888cb2721567251cff09)) +* **$pandas:** multiple conditions filter ([b25b7a6](https://github.com/johnnymillergh/python_boilerplate/commit/b25b7a63d755d8412023d03daf0c47b4eaab2bb1)) +* move all artifacts under `build` directory ([50e6c46](https://github.com/johnnymillergh/python_boilerplate/commit/50e6c46b8d3ff5303aaf17b8b0a93a3391addf11)) + + +### Performance Improvements + +* **$profiling:** replace process_time() -> perf_counter() ([99ca066](https://github.com/johnnymillergh/python_boilerplate/commit/99ca066915c76498394fcbb90d50295577b98c16)) +* **$profiling:** user `process_time()` for time profiling ([a55a4c7](https://github.com/johnnymillergh/python_boilerplate/commit/a55a4c7583f319a89626e18972bd0c0e1436c2fa)) +* **$pytest:** distribute tests across multiple CPUs to speed up test execution ([b16b2db](https://github.com/johnnymillergh/python_boilerplate/commit/b16b2db587e99c1f253dd351218e42d4d60f17fd)) + + + # [3.0.0](https://github.com/johnnymillergh/muscle-and-fitness-server/compare/2.0.0...3.0.0) (2022-09-06) diff --git a/Pipfile b/Pipfile index 68a0582..06e2fa6 100644 --- a/Pipfile +++ b/Pipfile @@ -45,8 +45,15 @@ matplotlib = "==3.5.3" # Black is the uncompromising Python code formatter. By using it, you agree to cede control over minutiae of # hand-formatting. https://pypi.org/project/black/ black = "==22.6.0" +# Naming Convention checker for Python. https://github.com/PyCQA/pep8-naming +pep8-naming = "==0.13.2" # Linting & toolkit for checking your code base against coding style (PEP8). https://pypi.org/project/flake8/ flake8 = "==5.0.4" +# https://github.com/DmytroLitvinov/awesome-flake8-extensions +flake8-quotes = "==3.3.1" +flake8-print = "==5.0.0" +flake8-use-fstring = "==1.4" +flake8-comprehensions = "==3.10.0" # isort your imports, so you don't have to. https://pypi.org/project/isort/ isort = "==5.10.1" # Add type annotations to your Python programs, and use mypy to type check them. https://pypi.org/project/mypy/ @@ -60,5 +67,9 @@ pytest = "==7.1.2" pytest-mock = "==3.8.2" # This plugin produces coverage reports. https://pypi.org/project/pytest-cov/ pytest-cov = "==3.0.0" +# Plugin for generating HTML reports for pytest results. https://github.com/pytest-dev/pytest-html/ +pytest-html = "==3.1.1" +# pytest xdist plugin for distributed testing and loop-on-failing modes. https://github.com/pytest-dev/pytest-xdist/ +pytest-xdist = "==2.5.0" # Call stack profiler for Python. Shows you why your code is slow! https://github.com/joerick/pyinstrument pyinstrument = "==4.3.0" diff --git a/Pipfile.lock b/Pipfile.lock index acc16cf..59425d4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f7908690051d2ccc468e977b8d4f14034bc0ecd6446f2eb2a2713975131cba4c" + "sha256": "999de30a6d9f694642097edc5bd379fa6e63c0f91fadee02371913d06e751abb" }, "pipfile-spec": 6, "requires": { @@ -24,14 +24,6 @@ "index": "pypi", "version": "==1.2.2" }, - "colorama": { - "hashes": [ - "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da", - "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4" - ], - "markers": "sys_platform == 'win32'", - "version": "==0.4.5" - }, "cycler": { "hashes": [ "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3", @@ -50,11 +42,11 @@ }, "fonttools": { "hashes": [ - "sha256:4606e1a88ee1f6699d182fea9511bd9a8a915d913eab4584e5226da1180fcce7", - "sha256:fff6b752e326c15756c819fe2fe7ceab69f96a1dbcfe8911d0941cdb49905007" + "sha256:88d48ef24486137c864dc56707b4b54ef8a97ab9162c2721ec61434baf1c4d13", + "sha256:b6d86ffd0a5f83d3da6a34d5f99a90398638e423cd6a8d93c5808af703432c7f" ], "markers": "python_version >= '3.7'", - "version": "==4.37.1" + "version": "==4.37.2" }, "jinja2": { "hashes": [ @@ -426,24 +418,9 @@ ], "index": "pypi", "version": "==8.0.1" - }, - "win32-setctime": { - "hashes": [ - "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2", - "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad" - ], - "markers": "sys_platform == 'win32'", - "version": "==1.1.0" } }, "develop": { - "atomicwrites": { - "hashes": [ - "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11" - ], - "markers": "sys_platform == 'win32'", - "version": "==1.4.1" - }, "attrs": { "hashes": [ "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", @@ -497,14 +474,6 @@ "markers": "python_version >= '3.7'", "version": "==8.1.3" }, - "colorama": { - "hashes": [ - "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da", - "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4" - ], - "markers": "sys_platform == 'win32'", - "version": "==0.4.5" - }, "coverage": { "extras": [ "toml" @@ -571,6 +540,14 @@ ], "version": "==0.3.6" }, + "execnet": { + "hashes": [ + "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5", + "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.9.0" + }, "filelock": { "hashes": [ "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc", @@ -587,13 +564,43 @@ "index": "pypi", "version": "==5.0.4" }, + "flake8-comprehensions": { + "hashes": [ + "sha256:181158f7e7aa26a63a0a38e6017cef28c6adee71278ce56ce11f6ec9c4905058", + "sha256:dad454fd3d525039121e98fa1dd90c46bc138708196a4ebbc949ad3c859adedb" + ], + "index": "pypi", + "version": "==3.10.0" + }, + "flake8-print": { + "hashes": [ + "sha256:76915a2a389cc1c0879636c219eb909c38501d3a43cc8dae542081c9ba48bdf9", + "sha256:84a1a6ea10d7056b804221ac5e62b1cee1aefc897ce16f2e5c42d3046068f5d8" + ], + "index": "pypi", + "version": "==5.0.0" + }, + "flake8-quotes": { + "hashes": [ + "sha256:633adca6fb8a08131536af0d750b44d6985b9aba46f498871e21588c3e6f525a" + ], + "index": "pypi", + "version": "==3.3.1" + }, + "flake8-use-fstring": { + "hashes": [ + "sha256:6550bf722585eb97dffa8343b0f1c372101f5c4ab5b07ebf0edd1c79880cdd39" + ], + "index": "pypi", + "version": "==1.4" + }, "identify": { "hashes": [ - "sha256:962d9bec27ccd1fcceff9b11f8c635afeb163ea3d8b6b30d8f1eee37ec7fac47", - "sha256:b020e876cec2b11dadb3324fa0427eb744b7d66ef19ac579a748dfff774b6dcf" + "sha256:322a5699daecf7c6fd60e68852f36f2ecbb6a36ff6e6e973e0d2bb6fca203ee6", + "sha256:ef78c0d96098a3b5fe7720be4a97e73f439af7cf088ebf47b620aeaa10fadf97" ], "markers": "python_version >= '3.7'", - "version": "==2.5.4" + "version": "==2.5.5" }, "iniconfig": { "hashes": [ @@ -678,6 +685,14 @@ "markers": "python_version >= '3.7'", "version": "==0.10.1" }, + "pep8-naming": { + "hashes": [ + "sha256:59e29e55c478db69cffbe14ab24b5bd2cd615c0413edf790d47d3fb7ba9a4e23", + "sha256:93eef62f525fd12a6f8c98f4dcc17fa70baae2f37fa1f73bec00e3e44392fa48" + ], + "index": "pypi", + "version": "==0.13.2" + }, "platformdirs": { "hashes": [ "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", @@ -806,6 +821,30 @@ "index": "pypi", "version": "==3.0.0" }, + "pytest-forked": { + "hashes": [ + "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e", + "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8" + ], + "markers": "python_version >= '3.6'", + "version": "==1.4.0" + }, + "pytest-html": { + "hashes": [ + "sha256:3ee1cf319c913d19fe53aeb0bc400e7b0bc2dbeb477553733db1dad12eb75ee3", + "sha256:b7f82f123936a3f4d2950bc993c2c1ca09ce262c9ae12f9ac763a2401380b455" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pytest-metadata": { + "hashes": [ + "sha256:39261ee0086f17649b180baf2a8633e1922a4c4b6fcc28a2de7d8127a82541bf", + "sha256:fcd2f416f15be295943527b3c8ba16a44ae5a7141939c90c3dc5ce9d167cf2a5" + ], + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==2.0.2" + }, "pytest-mock": { "hashes": [ "sha256:77f03f4554392558700295e05aed0b1096a20d4a60a4f3ddcde58b0c31c8fca2", @@ -814,8 +853,17 @@ "index": "pypi", "version": "==3.8.2" }, + "pytest-xdist": { + "hashes": [ + "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf", + "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65" + ], + "index": "pypi", + "version": "==2.5.0" + }, "pyyaml": { "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", @@ -827,26 +875,32 @@ "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" ], @@ -874,7 +928,7 @@ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], - "markers": "python_full_version < '3.11.0a7'", + "markers": "python_version < '3.11'", "version": "==2.0.1" }, "typing-extensions": { @@ -887,11 +941,11 @@ }, "virtualenv": { "hashes": [ - "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782", - "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22" + "sha256:227ea1b9994fdc5ea31977ba3383ef296d7472ea85be9d6732e42a91c04e80da", + "sha256:d07dfc5df5e4e0dbc92862350ad87a36ed505b978f6c39609dc489eadd5b0d27" ], "markers": "python_version >= '3.6'", - "version": "==20.16.4" + "version": "==20.16.5" } } } diff --git a/README.md b/README.md index 9c96cc2..c123d33 100644 --- a/README.md +++ b/README.md @@ -23,43 +23,45 @@ Here is the highlights of **python_boilerplate**: `Python` - [![Python](https://img.shields.io/badge/Python-v3.10.6-blue)](https://www.python.org/downloads/release/python-3106/) - `Pipenv` is to build and compile the project + `Pipenv` is to build and compile the project. -2. Highly customizable data analysis with [pandas](https://pandas.pydata.org/), enhanced array operation with [NumPy](https://numpy.org/). Supports CSV, excel, JSON and so on +2. Highly customizable data analysis with [pandas](https://pandas.pydata.org/), enhanced array operation with [NumPy](https://numpy.org/). Supports CSV, excel, JSON and so on. -3. Data persistence with [peewee](http://docs.peewee-orm.com/en/latest/), [SQLite3](https://sqlite.org/index.html) as local database +3. Data persistence with [peewee](http://docs.peewee-orm.com/en/latest/), [SQLite3](https://sqlite.org/index.html) as local database. -4. Simple and flexible retry with [Tenacity](https://github.com/jd/tenacity) +4. Simple and flexible retry with [Tenacity](https://github.com/jd/tenacity). -5. Environment variable and configuration with [pyhocon](https://pythonhosted.org/pyhocon/_modules/pyhocon.html). Read `${ENVIRONMENT_VARIABLE}` when startup +5. Environment variable and configuration with [pyhocon](https://pythonhosted.org/pyhocon/_modules/pyhocon.html). Read `${ENVIRONMENT_VARIABLE}` when startup. -6. Sensible and human-friendly approach to creating, manipulating, formatting and converting dates, times and timestamps with [Arrow](https://pypi.org/project/arrow/) +6. Sensible and human-friendly approach to creating, manipulating, formatting and converting dates, times and timestamps with [Arrow](https://pypi.org/project/arrow/). -7. Generate fake data with [Faker](https://pypi.org/project/Faker/) +7. Generate fake data with [Faker](https://pypi.org/project/Faker/). -8. Testing with [pytest](https://docs.pytest.org/en/latest/), integrating [pytest-mock](https://pypi.org/project/pytest-mock/) for mocking, [pytest-cov](https://pypi.org/project/pytest-cov/) for code coverage analysis and [pyinstrument](https://github.com/joerick/pyinstrument) for Python stack profiler +8. Customized function decorator `@async_function` to enable function to run asynchronously; `@peewee_table` class decorator to register ORM tables; `@elapsed_time(level="INFO")` to profile a function elapsed time. -9. Formatting with [black](https://github.com/psf/black) +9. Testing with [pytest](https://docs.pytest.org/en/latest/), integrating [pytest-mock](https://pypi.org/project/pytest-mock/) for mocking, [pytest-cov](https://pypi.org/project/pytest-cov/) for code coverage analysis and [pyinstrument](https://github.com/joerick/pyinstrument) for Python stack profiler. -10. Import sorting with [isort](https://github.com/timothycrosley/isort) +10. Formatting with [black](https://github.com/psf/black). -11. Static typing with [mypy](http://mypy-lang.org/) +11. Import sorting with [isort](https://github.com/timothycrosley/isort). -12. Linting with [flake8](http://flake8.pycqa.org/en/latest/) +12. Static typing with [mypy](http://mypy-lang.org/). -13. Git hooks that run all the above with [pre-commit](https://pre-commit.com/) +13. Linting with [flake8](http://flake8.pycqa.org/en/latest/). -14. Deployment ready with [Docker](https://docker.com/) +14. Git hooks that run all the above with [pre-commit](https://pre-commit.com/). -15. Continuous Integration with [GitHub Actions](https://github.com/features/actions) +15. Deployment ready with [Docker](https://docker.com/). -16. Loguru logging configuration. Log sample is like, +16. Continuous Integration with [GitHub Actions](https://github.com/features/actions). + +17. Loguru logging configuration. Log sample is like, ``` - 2022-09-06 10:02:06.716 | ⚠️ WARNING | MainThread | python_boilerplate.repository.model.base_model.:24 - SQLite database created. Path: [D:\Projects\PyCharmProjects\python_boilerplate\data\python_boilerplate.db], - 2022-09-06 10:02:06.718 | ℹ️ INFO | MainThread | python_boilerplate.common.orm.peewee_table:16 - Registering peewee table: StartupLog - 2022-09-06 10:02:06.719 | 🐞 DEBUG | MainThread | peewee.execute_sql:3185 - ('CREATE TABLE IF NOT EXISTS "startup_log" ("id" INTEGER NOT NULL PRIMARY KEY, "current_user" VARCHAR(50) NOT NULL, "host" VARCHAR(50) NOT NULL, "command_line" TEXT NOT NULL, "current_working_directory" TEXT NOT NULL, "startup_time" DATETIME NOT NULL, "created_by" VARCHAR(50) NOT NULL, "created_time" DATETIME NOT NULL, "modified_by" VARCHAR(50) NOT NULL, "modified_time" DATETIME NOT NULL)', []) - 2022-09-06 10:02:06.882 | ℹ️ INFO | MainThread | python_boilerplate.:35 - Application [python_boilerplate] started + 2022-09-17 14:13:52.385 | ⚠️ WARNING | 6860 | MainThread | python_boilerplate.repository.model.base_model.:24 - SQLite database created. Path: [/Users/johnny/Projects/PyCharmProjects/python_boilerplate/data/python_boilerplate.db], + 2022-09-17 14:13:52.386 | ℹ️ INFO | 6860 | MainThread | python_boilerplate.common.orm.peewee_table:16 - Registering peewee table: StartupLog + 2022-09-17 14:13:52.387 | 🐞 DEBUG | 6860 | MainThread | peewee.execute_sql:3185 - ('CREATE TABLE IF NOT EXISTS "startup_log" ("id" INTEGER NOT NULL PRIMARY KEY, "current_user" VARCHAR(50) NOT NULL, "host" VARCHAR(50) NOT NULL, "command_line" TEXT NOT NULL, "current_working_directory" TEXT NOT NULL, "startup_time" DATETIME NOT NULL, "created_by" VARCHAR(50) NOT NULL, "created_time" DATETIME NOT NULL, "modified_by" VARCHAR(50) NOT NULL, "modified_time" DATETIME NOT NULL)', []) + 2022-09-17 14:13:52.530 | ℹ️ INFO | 6860 | MainThread | python_boilerplate.:53 - Started python_boilerplate in 0.117 seconds (117.26 ms) ``` ## Usage @@ -82,7 +84,7 @@ Here is the highlights of **python_boilerplate**: # Install pipx if pipenv and cookiecutter are not installed $ python3 -m pip install pipx $ python3 -m pipx ensurepath - + # Install pipenv using pipx $ pipx install pipenv ``` @@ -102,6 +104,31 @@ Here is the highlights of **python_boilerplate**: ## Useful Commands +### Run Python Module + +```shell +$ python3 -m python_boilerplate +``` + +### Run Python Script + +**Append your project’s root directory to** `PYTHONPATH` — In any environment you wish to run your Python application such as Docker, vagrant or your virtual environment i.e. in bin/activate, run the below command: + +> [How to Fix ModuleNotFoundError and ImportError](https://towardsdatascience.com/how-to-fix-modulenotfounderror-and-importerror-248ce5b69b1c) + +```shell +$ PYTHONPATH="${PYTHONPATH}:/Users/johnny/Projects/PyCharmProjects/python_boilerplate/python_boilerplate" python3 python_boilerplate/__main__.py +``` + +### ~~Package with [PyInstaller](https://pyinstaller.org/en/latest/usage.html?highlight=pythonpath#using-pyinstaller)~~ + +```shell +$ pipenv run pyinstaller --console \ +--add-data "python_boilerplate/resources/*:python_boilerplate/resources" \ +--name main \ +--clean --noconfirm python_boilerplate/demo/pandas_usage.py +``` + ### Run Unit Tests Run with pytest, analyze code coverage, generate HTML code coverage reports, fail the test if coverage percentage is unser 85% @@ -145,7 +172,7 @@ Feel free to dive in! [Open an issue](https://github.com/johnnymillergh/python_b ### Contributors -This project exists thanks to all the people who contribute. +This project exists thanks to all the people who contribute. - Johnny Miller [[@johnnymillergh](https://github.com/johnnymillergh)] - … diff --git a/python_boilerplate/__init__.py b/python_boilerplate/__init__.py index d753055..552d407 100644 --- a/python_boilerplate/__init__.py +++ b/python_boilerplate/__init__.py @@ -1,5 +1,9 @@ import atexit +import os +import platform import sys +import time +from pathlib import Path from loguru import logger @@ -25,6 +29,12 @@ ) from python_boilerplate.repository.trace_log_repository import retain_trace_log +__start_time = time.perf_counter() +logger.info( + f"Starting {get_module_name()} using Python {platform.python_version()} on " + f"{platform.node()} with PID {os.getpid()} ({Path(__file__).parent})" +) + # Configuration application_configure() loguru_configure() @@ -32,7 +42,6 @@ # Initialization __init__() -logger.info(f"Application [{get_module_name()}] started") # Saving startup log # Cannot save startup log in parallel, because the ThreadPoolExecutor won't be able to start another future @@ -40,16 +49,26 @@ # executor.submit(save, StartupLog(command_line=" ".join(sys.argv))).add_done_callback(done_callback) save(StartupLog(command_line=" ".join(sys.argv))) +__elapsed = time.perf_counter() - __start_time +logger.info( + f"Started {get_module_name()} in {round(__elapsed, 3)} seconds ({round(__elapsed * 1000, 2)} ms)" +) + @atexit.register def finalize() -> None: """ Register `finalize()` function to be executed upon normal program termination. """ - logger.warning("Cleaning up…") + logger.warning(f"Stopping {get_module_name()}, releasing system resources") # Retain logs, in case the size of the SQLite database will be increasing like crazy. retain_startup_log() retain_trace_log() # Shutdown tread pool and other connections thread_pool_cleanup() email_cleanup() + __end_elapsed = time.perf_counter() - __start_time + logger.info( + f"Stopped {get_module_name()}, running for {round(__end_elapsed, 3)} seconds " + f"({round(__end_elapsed * 1000, 2)} ms) in total" + ) diff --git a/python_boilerplate/common/asynchronization.py b/python_boilerplate/common/asynchronization.py new file mode 100644 index 0000000..9cb3952 --- /dev/null +++ b/python_boilerplate/common/asynchronization.py @@ -0,0 +1,40 @@ +import functools +from typing import Callable + +from loguru import logger + +from python_boilerplate.configuration.thread_pool_configuration import ( + done_callback, + executor, +) + + +def async_function(func: Callable): + """ + The decorator to run function in thread pool. The return value of decorated function will be + `concurrent.futures._base.Future`. + + Usage: decorate the function with `@async_function` + + https://stackoverflow.com/questions/37203950/decorator-for-extra-thread + + :param func: function to run in thread pool + """ + + @functools.wraps(func) + def wrapped(*arg, **kwarg): + if arg and not kwarg: + submitted_future = executor.submit(func, *arg) + elif not arg and kwarg: + submitted_future = executor.submit(func, **kwarg) + elif arg and kwarg: + submitted_future = executor.submit(func, *arg, **kwarg) + else: + submitted_future = executor.submit(func) + submitted_future.add_done_callback(done_callback) + logger.debug( + f"Submitted future task {submitted_future} to run [{func.__qualname__}()] asynchronously" + ) + return submitted_future + + return wrapped diff --git a/python_boilerplate/common/common_function.py b/python_boilerplate/common/common_function.py index edd2839..d3a19ec 100644 --- a/python_boilerplate/common/common_function.py +++ b/python_boilerplate/common/common_function.py @@ -1,7 +1,8 @@ +import getpass import os from datetime import date, datetime from pathlib import Path -from typing import Any +from typing import Any, Final from loguru import logger @@ -11,8 +12,8 @@ # Path Correspondence to tools in the os module # https://docs.python.org/3/library/pathlib.html#correspondence-to-tools-in-the-os-module -PROJECT_ROOT_PATH = Path(__file__).parent.parent.parent -MODULE_ROOT_PATH = Path(__file__).parent.parent +PROJECT_ROOT_PATH: Final = Path(__file__).parent.parent.parent +MODULE_ROOT_PATH: Final = Path(__file__).parent.parent def get_data_dir(sub_path="") -> Path: @@ -63,8 +64,8 @@ def get_login_user() -> str: :return: the username """ try: - return os.getlogin() - except OSError as ex: + return getpass.getuser() + except Exception as ex: logger.error( f"Failed to get current login user, falling back to `default_user`. {ex}" ) diff --git a/python_boilerplate/common/profiling.py b/python_boilerplate/common/profiling.py index eb111db..b3724b6 100644 --- a/python_boilerplate/common/profiling.py +++ b/python_boilerplate/common/profiling.py @@ -15,19 +15,19 @@ def elapsed_time(level="INFO"): https://stackoverflow.com/questions/12295974/python-decorators-just-syntactic-sugar - :param level: logging level, default is `INFO` + :param level: logging level, default is `INFO`. Available values: "TRACE", "DEBUG", "INFO", "WARNING", "ERROR" """ def elapsed_time_wrapper(func: Callable): @functools.wraps(func) def wrapped(*arg, **kwarg): - start_time = time.time() + start_time = time.perf_counter() return_value = func(*arg, **kwarg) - end_time = time.time() + elapsed = time.perf_counter() - start_time logger.log( level, - f"{func.__qualname__}() -> elapsed time: {round(end_time - start_time, 4)}s " - f"({round((end_time - start_time) * 1000, 2)}ms)", + f"{func.__qualname__}() -> elapsed time: {round(elapsed, 4)} s " + f"({round(elapsed * 1000, 2)} ms)", ) return return_value diff --git a/python_boilerplate/configuration/loguru_configuration.py b/python_boilerplate/configuration/loguru_configuration.py index 34086f5..f5e49f8 100644 --- a/python_boilerplate/configuration/loguru_configuration.py +++ b/python_boilerplate/configuration/loguru_configuration.py @@ -10,6 +10,7 @@ _message_format = ( "{time:YYYY-MM-DD HH:mm:ss.SSS} | " "{level.icon} {level: <8} | " + "{process.id} | " "{thread.name: <15} | " "{name}.{function}:{line} - " "{message}" diff --git a/python_boilerplate/configuration/thread_pool_configuration.py b/python_boilerplate/configuration/thread_pool_configuration.py index 9cd0749..7335c5c 100644 --- a/python_boilerplate/configuration/thread_pool_configuration.py +++ b/python_boilerplate/configuration/thread_pool_configuration.py @@ -39,16 +39,19 @@ def configure() -> None: ) -@elapsed_time() +# noinspection PyProtectedMember +@elapsed_time("WARNING") def cleanup() -> None: """ Clean up thread pool. """ logger.warning( - f"Thread pool executor is being shutdown: {executor}, pending: {executor._work_queue.qsize()} jobs, threads: {len(executor._threads)}" + f"Thread pool executor is being shutdown: {executor}, pending: {executor._work_queue.qsize()} jobs, " + f"threads: {len(executor._threads)}" ) executor.shutdown() # noinspection PyProtectedMember logger.warning( - f"Thread pool executor has been shutdown: {executor}, pending: {executor._work_queue.qsize()} jobs, threads: {len(executor._threads)}" + f"Thread pool executor has been shutdown: {executor}, pending: {executor._work_queue.qsize()} jobs, " + f"threads: {len(executor._threads)}" ) diff --git a/python_boilerplate/demo/pandas_usage.py b/python_boilerplate/demo/pandas_usage.py index d71d0e8..f824e03 100644 --- a/python_boilerplate/demo/pandas_usage.py +++ b/python_boilerplate/demo/pandas_usage.py @@ -1,10 +1,13 @@ -from typing import Final +from concurrent.futures import Future, wait +from typing import Any, Final import numpy as np import pandas as pd +from faker import Faker from loguru import logger from pandas import DataFrame, DatetimeIndex, Series +from python_boilerplate.common.asynchronization import async_function from python_boilerplate.common.common_function import get_data_dir, get_resources_dir from python_boilerplate.common.profiling import elapsed_time from python_boilerplate.common.trace import trace @@ -21,6 +24,8 @@ video_games: Final = pd.read_csv(video_games_path) logger.info(f"Done reading CSV, file: [{video_games_path}], rows: {len(video_games)}") +faker = Faker() + def pandas_data_structure_series() -> Series: return pd.Series([1, 3, 5, np.nan, 6, 8]) @@ -31,7 +36,7 @@ def pandas_data_structure_date_range() -> DatetimeIndex: @trace -@elapsed_time() +@elapsed_time("INFO") def look_for_sony_published_games() -> DataFrame: all_columns = set(video_games) selected_columns = { @@ -43,14 +48,17 @@ def look_for_sony_published_games() -> DataFrame: } dropped_columns = list(all_columns - selected_columns) sony_published: Final = video_games[ - video_games["Metadata.Publishers"] == "Sony" + (video_games["Metadata.Publishers"] == "Sony") + & (video_games["Features.Max Players"] > 1) + & (video_games["Title"].str.contains("t")) ].drop(columns=dropped_columns) release_year: Final = "Release.Year" sony_games_release_year: Final = sony_published[release_year] min_release_year = sony_games_release_year.sort_values().min() max_release_year = sony_games_release_year.sort_values().max() logger.info( - f"From {min_release_year} to {max_release_year}, Sony has published {len(sony_published)} games" + f"From {min_release_year} to {max_release_year}, Sony has published {len(sony_published)} games, " + f"those are multi-player, title with 't'" ) game_release_each_year: Final[Series] = ( sony_published.groupby(release_year)[release_year] @@ -63,4 +71,55 @@ def look_for_sony_published_games() -> DataFrame: return sony_published -look_for_sony_published_games() +@async_function +@elapsed_time("DEBUG") +def generate_random_data(row_count: int) -> DataFrame: + rows: list[dict[str, Any]] = [] + for _ in range(row_count): + rows.append( + { + "full_name": faker.name(), + "age": faker.random.randint(18, 100), + "phone_number": faker.phone_number(), + "address": faker.address(), + "zipcode": faker.zipcode(), + "country": faker.country(), + } + ) + return pd.DataFrame(rows) + + +# noinspection PyTypeChecker +@elapsed_time("DEBUG") +def submit_parallel_tasks() -> list[DataFrame]: + futures: list[Future] = [] + for _ in range(5): + futures.append(generate_random_data(5000)) + wait(futures) + logger.info(f"All {len(futures)} tasks has done") + return [future.result() for future in futures] + + +@elapsed_time("DEBUG") +def merge_results(dataframes: list[DataFrame]) -> DataFrame: + result_list: DataFrame = pd.DataFrame( + columns=["full_name", "age", "phone_number", "address", "zipcode", "country"] + ) + for dataframe in dataframes: + result_list = pd.concat([result_list, dataframe]) + return result_list + + +def data_generation(): + futures = submit_parallel_tasks() + result_data_pd: DataFrame = merge_results(futures) + logger.info(f"Finished merging data\n{result_data_pd}") + random_data_path = get_data_dir() / "random_data.csv" + result_data_pd.to_csv(random_data_path, index=False) + logger.info( + f"Done writing data file [{random_data_path}], rows: {len(result_data_pd)}" + ) + + +if __name__ == "__main__": + look_for_sony_published_games() diff --git a/setup.cfg b/setup.cfg index daeddb1..d235122 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,20 +1,37 @@ +[metadata] +name = python_boilerplate +version = 3.0.0 +description = A boilerplate project for Python. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/johnnymillergh/python_boilerplate +author = Johnny Miller +author_email = johnnysviva@outlook.com +license = Apache License 2.0 +license_file = LICENSE + [flake8] ignore = E203, E266, E501, W503 ; PEP 8 - Maximum Line Length, https://peps.python.org/pep-0008/#maximum-line-length max-line-length = 88 max-complexity = 18 select = B,C,E,F,W,T4 +inline-quotes = " +multiline-quotes = " +docstring-quotes = " [isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +line_length = 88 [mypy] -files=python_boilerplate,tests -ignore_missing_imports=true +files = python_boilerplate,tests +cache_dir = ./build/.mypy_cache +ignore_missing_imports = true [tool:pytest] -testpaths=tests/ +testpaths = tests/ +cache_dir = ./build/.pytest_cache diff --git a/tests/common/test_asynchronization.py b/tests/common/test_asynchronization.py new file mode 100644 index 0000000..711ac16 --- /dev/null +++ b/tests/common/test_asynchronization.py @@ -0,0 +1,66 @@ +from concurrent.futures import Future +from time import sleep + +import arrow +from loguru import logger + +from python_boilerplate.common.asynchronization import async_function + + +@async_function +def an_async_function(param1: str, param2: str): + sleep(1) + logger.info(f"Function's param1={param1}, param2={param2}") + return f"Hello, got param1={param1}, param2={param2}" + + +@async_function +def another_async_function_without_args(): + sleep(1) + logger.info(f"Hello from {another_async_function_without_args.__qualname__}") + + +def test_async_function_expected_no_errors(): + try: + a_future: Future = an_async_function( + arrow.now().__str__(), arrow.now().shift(days=-1).__str__() + ) + assert a_future is not None + result = a_future.result() + assert len(result) > 0 + logger.info(f"Got future result: {result}") + except Exception as ex: + assert False, f"{an_async_function} raised an exception {ex}" + + +def test_async_function_pass_kwarg_expected_no_errors(): + try: + a_future: Future = an_async_function( + param1=arrow.now().__str__(), param2=arrow.now().shift(days=-1).__str__() + ) + assert a_future is not None + result = a_future.result() + assert len(result) > 0 + logger.info(f"Got future result: {result}") + except Exception as ex: + assert False, f"{an_async_function} raised an exception {ex}" + + +def test_async_function_pass_arg_kwarg_expected_no_errors(): + try: + a_future: Future = an_async_function( + arrow.now().__str__(), param2=arrow.now().shift(days=-1).__str__() + ) + assert a_future is not None + result = a_future.result() + assert len(result) > 0 + logger.info(f"Got future result: {result}") + except Exception as ex: + assert False, f"{an_async_function} raised an exception {ex}" + + +def test_another_async_function_without_args_expected_no_errors(): + try: + another_async_function_without_args() + except Exception as ex: + assert False, f"{another_async_function_without_args} raised an exception {ex}" diff --git a/tests/conftest.py b/tests/conftest.py index ba03317..f1e3c49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,18 @@ from loguru import logger from pyinstrument import Profiler -TESTS_ROOT = Path.cwd() +from python_boilerplate.common.common_function import get_module_name + +PROJECT__ROOT = Path(__file__).parent.parent + + +def pytest_html_report_title(report): + """ + pytest-html title configuration. + + https://pytest-html.readthedocs.io/en/latest/user_guide.html#user-guide + """ + report.title = f"Pytest Report of {get_module_name()}" @pytest.fixture(autouse=True) @@ -16,8 +27,7 @@ def auto_profile(request): https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-pytest-tests """ - # noinspection PyPep8Naming - PROFILE_ROOT = TESTS_ROOT / ".profiles" + profile_root = PROJECT__ROOT / "build/.profiles" logger.info("Starting to profile Pytest unit tests...") # Turn profiling on profiler = Profiler() @@ -27,7 +37,7 @@ def auto_profile(request): profiler.stop() node: Node = request.node - profile_html_path = PROFILE_ROOT / f"{node.path.parent.relative_to(TESTS_ROOT)}" + profile_html_path = profile_root / f"{node.path.parent.relative_to(PROJECT__ROOT)}" if not profile_html_path.exists(): # If parents is false (the default), a missing parent raises FileNotFoundError. # If exist_ok is false (the default), FileExistsError is raised if the target directory already exists. diff --git a/tests/demo/test_pandas_usage.py b/tests/demo/test_pandas_usage.py index a5e62e0..043dad6 100644 --- a/tests/demo/test_pandas_usage.py +++ b/tests/demo/test_pandas_usage.py @@ -6,6 +6,7 @@ from pandas import DatetimeIndex, Series from python_boilerplate.demo.pandas_usage import ( + data_generation, look_for_sony_published_games, pandas_data_structure_date_range, pandas_data_structure_series, @@ -57,5 +58,12 @@ def test_pandas_reading_csv() -> None: def test_look_for_sony_published_games(): sony_published_games = look_for_sony_published_games() assert sony_published_games is not None - assert len(sony_published_games) == 60 + assert len(sony_published_games) == 9 assert Path(sony_published_video_games_path).exists(), "CSV file NOT exists!" + + +def test_data_generation(): + try: + data_generation() + except Exception as ex: + assert False, f"{data_generation} raised an exception {ex}"