diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c9ae6fe..89cdba0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: matrix: os: [ubuntu-20.04] python-version: ['3.8'] - toxenv: ["py38", "quality", "docs"] + toxenv: ["quality", "docs", "django32", "django42"] steps: - uses: actions/checkout@v4 @@ -37,7 +37,7 @@ jobs: run: tox - name: Run coverage - if: matrix.python-version == '3.8' && matrix.toxenv == 'py38' + if: matrix.python-version == '3.8' && matrix.toxenv == 'django42' uses: codecov/codecov-action@v3 with: flags: unittests diff --git a/Makefile b/Makefile index 8db4de10..a4f6bd28 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: clean compile_translations coverage diff_cover docs dummy_translations \ - extract_translations fake_translations help \ - quality requirements selfcheck test test-all upgrade validate +.PHONY: clean clean_tox compile_translations coverage diff_cover docs dummy_translations \ + extract_translations fake_translations help pii_check pull_translations push_translations \ + quality requirements selfcheck test test-all upgrade validate install_transifex_client .DEFAULT_GOAL := help @@ -21,6 +21,9 @@ clean: ## remove generated byte code, coverage reports, and build artifacts rm -fr dist/ rm -fr *.egg-info +clean_tox: ## clear tox requirements cache + rm -fr .tox + coverage: clean ## generate and view HTML coverage report pytest --cov-report html $(BROWSER)htmlcov/index.html @@ -32,42 +35,36 @@ docs: ## generate Sphinx HTML documentation, including API docs # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade. PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS) -COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt -.PHONY: $(COMMON_CONSTRAINTS_TXT) -$(COMMON_CONSTRAINTS_TXT): - wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)" - upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: $(COMMON_CONSTRAINTS_TXT) ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in - pip install -r requirements/pip-tools.txt +upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in + pip install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! - $(PIP_COMPILE) --allow-unsafe --rebuild -o requirements/pip.txt requirements/pip.in + $(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in $(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in - pip install -r requirements/pip.txt - pip install -r requirements/pip-tools.txt + pip install -qr requirements/pip.txt + pip install -qr requirements/pip-tools.txt $(PIP_COMPILE) -o requirements/base.txt requirements/base.in $(PIP_COMPILE) -o requirements/test.txt requirements/test.in $(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in $(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in $(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in $(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in + # Let tox control the Django version for tests + sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp + mv requirements/test.tmp requirements/test.txt quality: ## check coding style with pycodestyle and pylint - touch tests/__init__.py - pylint openedx_events tests test_utils *.py - rm tests/__init__.py - pycodestyle openedx_events tests *.py - pydocstyle openedx_events tests *.py - isort --check-only --diff --recursive tests test_utils openedx_events *.py test_settings.py - python setup.py bdist_wheel - twine check dist/* - make selfcheck - - -requirements: ## install development environment requirements + tox -e quality + +pii_check: ## check for PII annotations on all Django models + tox -e pii_check + +piptools: ## install pinned version of pip-compile and pip-sync pip install -r requirements/pip.txt pip install -r requirements/pip-tools.txt - pip-sync requirements/dev.txt requirements/private.* + +requirements: clean_tox piptools ## install development environment requirements + pip-sync -q requirements/dev.txt requirements/private.* test: clean ## run tests in the current virtualenv pytest @@ -75,10 +72,53 @@ test: clean ## run tests in the current virtualenv diff_cover: test ## find diff lines that need test coverage diff-cover coverage.xml -test-all: quality ## run tests on every supported Python/Django combination +test-all: quality pii_check ## run tests on every supported Python/Django combination tox + tox -e docs -validate: quality test ## run tests and quality checks +validate: quality pii_check test ## run tests and quality checks selfcheck: ## check that the Makefile is well-formed @echo "The Makefile is well-formed." + +## Localization targets + +extract_translations: ## extract strings to be translated, outputting .mo files + rm -rf docs/_build + cd {{cookiecutter.app_name}} && i18n_tool extract --no-segment + +compile_translations: ## compile translation files, outputting .po files for each supported language + cd {{cookiecutter.app_name}} && i18n_tool generate + +detect_changed_source_translations: + cd {{cookiecutter.app_name}} && i18n_tool changed + +ifeq ($(OPENEDX_ATLAS_PULL),) +pull_translations: ## Pull translations from Transifex + tx pull -t -a -f --mode reviewed --minimum-perc=1 +else +# Experimental: OEP-58 Pulls translations using atlas +pull_translations: + find {{cookiecutter.app_name}}/conf/locale -mindepth 1 -maxdepth 1 -type d -exec rm -r {} \; + atlas pull $(OPENEDX_ATLAS_ARGS) translations/{{cookiecutter.repo_name}}/{{cookiecutter.app_name}}/conf/locale:{{cookiecutter.app_name}}/conf/locale + python manage.py compilemessages + + @echo "Translations have been pulled via Atlas and compiled." +endif + +push_translations: ## push source translation files (.po) from Transifex + tx push -s + +dummy_translations: ## generate dummy translation (.po) files + cd {{cookiecutter.app_name}} && i18n_tool dummy + +build_dummy_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files + +validate_translations: build_dummy_translations detect_changed_source_translations ## validate translations + +install_transifex_client: ## Install the Transifex client + # Instaling client will skip CHANGELOG and LICENSE files from git changes + # so remind the user to commit the change first before installing client. + git diff -s --exit-code HEAD || { echo "Please commit changes first."; exit 1; } + curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash + git checkout -- LICENSE README.md ## overwritten by Transifex installer diff --git a/requirements/base.txt b/requirements/base.txt index ec212d87..bf871fa8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,7 +14,7 @@ click==8.1.7 # via edx-django-utils django==3.2.24 # via - # -c requirements/common_constraints.txt + # -c https://raw.githubusercontent.com/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in # django-crum # django-waffle diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt deleted file mode 100644 index 15aafb29..00000000 --- a/requirements/common_constraints.txt +++ /dev/null @@ -1,23 +0,0 @@ -# A central location for most common version constraints -# (across edx repos) for pip-installation. -# -# Similar to other constraint files this file doesn't install any packages. -# It specifies version constraints that will be applied if a package is needed. -# When pinning something here, please provide an explanation of why it is a good -# idea to pin this package across all edx repos, Ideally, link to other information -# that will help people in the future to remove the pin when possible. -# Writing an issue against the offending project and linking to it here is good. -# -# Note: Changes to this file will automatically be used by other repos, referencing -# this file from Github directly. It does not require packaging in edx-lint. - - -# using LTS django version -Django<4.0 - -# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. -# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html -elasticsearch<7.14.0 - -# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected -django-simple-history==3.0.0 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e157673e..09c2c45e 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -7,7 +7,9 @@ # link to other information that will help people in the future to remove the # pin when possible. Writing an issue against the offending project and # linking to it here is good. --c common_constraints.txt + +# Common constraints for openedx repos +-c https://raw.githubusercontent.com/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # Temporary solution since this version raises RecursionError for test_generate_avro_schemas.py # This should be removed once the issue is fixed with a new astroid release or with a test_generate_avro_schemas.py diff --git a/requirements/dev.txt b/requirements/dev.txt index e74dcba5..9d4a1b05 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -71,7 +71,7 @@ cryptography==42.0.5 # via # -r requirements/quality.txt # secretstorage -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/quality.txt diff-cover==8.0.3 # via -r requirements/dev.in @@ -85,7 +85,7 @@ distlib==0.3.8 # virtualenv django==3.2.24 # via - # -c requirements/common_constraints.txt + # -c https://raw.githubusercontent.com/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # django-crum # django-waffle @@ -301,7 +301,7 @@ pyyaml==6.0.1 # via # -r requirements/quality.txt # code-annotations -readme-renderer==42.0 +readme-renderer==43.0 # via # -r requirements/quality.txt # twine diff --git a/requirements/doc.txt b/requirements/doc.txt index 42a816e3..7cde8e5f 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -46,11 +46,11 @@ coverage[toml]==7.4.3 # pytest-cov cryptography==42.0.5 # via secretstorage -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/test.txt django==3.2.24 # via - # -c requirements/common_constraints.txt + # -c https://raw.githubusercontent.com/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt # django-crum # django-waffle @@ -196,7 +196,7 @@ pyyaml==6.0.1 # via # -r requirements/test.txt # code-annotations -readme-renderer==42.0 +readme-renderer==43.0 # via twine requests==2.31.0 # via diff --git a/requirements/quality.txt b/requirements/quality.txt index 7d5a52fd..530c2e19 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -43,13 +43,13 @@ coverage[toml]==7.4.3 # pytest-cov cryptography==42.0.5 # via secretstorage -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/test.txt dill==0.3.8 # via pylint django==3.2.24 # via - # -c requirements/common_constraints.txt + # -c https://raw.githubusercontent.com/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt # django-crum # django-waffle @@ -199,7 +199,7 @@ pyyaml==6.0.1 # via # -r requirements/test.txt # code-annotations -readme-renderer==42.0 +readme-renderer==43.0 # via twine requests==2.31.0 # via diff --git a/requirements/test.txt b/requirements/test.txt index 688922d4..d2486cbc 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -23,11 +23,10 @@ code-annotations==1.6.0 # via -r requirements/test.in coverage[toml]==7.4.3 # via pytest-cov -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/test.in -django==3.2.24 # via - # -c requirements/common_constraints.txt + # -c https://raw.githubusercontent.com/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt # django-crum # django-waffle diff --git a/tox.ini b/tox.ini index cd8a9dc6..ef491ef9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,12 @@ [tox] -envlist = py{38}, quality, docs, pii_check - +envlist = py{38}-django{32, 42}, quality, docs, pii_check [doc8] ; D001 = Line too long -ignore=D001 +ignore = D001 [pycodestyle] -exclude = .git,.tox +exclude = .git,.tox,migrations max-line-length = 120 max-doc-length = 120 @@ -21,7 +20,7 @@ max-doc-length = 120 ; D405 = Section name should be properly capitalized (numpy style) ; D406 = Section name should end with a newline (numpy style) ; D407 = Missing dashed underline after section (numpy style) -; D408 = Section underline should be in the line following the section’s name (numpy style) +; D408 = Section underline should be in the line following the section's name (numpy style) ; D409 = Section underline should match the length of its name (numpy style) ; D410 = Missing blank line after section (numpy style) ; D411 = Missing blank line before section (numpy style) @@ -29,38 +28,41 @@ max-doc-length = 120 ; D413 = Missing blank line after last section (numpy style) ; D414 = Section has no content (numpy style) ignore = D101,D200,D203,D212,D215,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 - +match-dir = (?!migrations) [pytest] DJANGO_SETTINGS_MODULE = test_utils.test_settings -addopts = --cov openedx_events --cov-report term-missing --cov-report xml +addopts = --cov openedx_events --cov tests --cov-report term-missing --cov-report xml norecursedirs = .* docs requirements site-packages [testenv] deps = + django32: Django>=3.2,<4.0 + django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt commands = - pytest openedx_events tests + python manage.py check + pytest {posargs} [testenv:docs] setenv = + DJANGO_SETTINGS_MODULE = test_utils.test_settings PYTHONPATH = {toxinidir} + # Adding the option here instead of as a default in the docs Makefile because that Makefile is generated by shpinx. SPHINXOPTS = -W allowlist_externals = make rm - twine deps = -r{toxinidir}/requirements/doc.txt commands = doc8 --ignore-path docs/_build README.rst docs rm -f docs/openedx_events.rst rm -f docs/modules.rst - # -e allows for overriding setting from the environment so that SPHINXOPTS gets picked up. make -e -C docs clean make -e -C docs html - python setup.py sdist bdist_wheel - twine check --strict dist/* + python -m build --wheel + twine check dist/* [testenv:quality] allowlist_externals = @@ -70,7 +72,13 @@ allowlist_externals = deps = -r{toxinidir}/requirements/quality.txt commands = - make quality + touch tests/__init__.py + pylint openedx_events tests test_utils manage.py setup.py + rm tests/__init__.py + pycodestyle openedx_events tests manage.py setup.py + pydocstyle openedx_events tests manage.py setup.py + isort --check-only --diff tests test_utils openedx_events manage.py setup.py + make selfcheck [testenv:pii_check] setenv =