Merge pull request #734 from linshokaku/makefile-utils

Make Command Calls in GitHub Workflows

.github/workflows/nightly-test-cpu.yml

@@ -25,13 +25,10 @@ jobs:
     - name: Install
       run: |
         pip install -U pip wheel
-        pip install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu
-        pip install pytest
-        pip install matplotlib tensorboard ipython ipywidgets pandas optuna onnx onnxruntime pytorch-ignite
-        pip install -v -e .
+        pip install -v -e .[test] --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu 
         # Test PPE is importable with minimum dependency
         python -c 'import pytorch_pfn_extras'
 
     - name: Test CPU only
       run: |
-        pytest -m "not gpu" tests
+        make cputest

.github/workflows/pretest-and-test.yml

@@ -31,30 +31,18 @@ jobs:
     - name: Install
       run: |
         pip install -U pip wheel
-        pip install 'torch==${{ matrix.torch }}' --extra-index-url https://download.pytorch.org/whl/cpu
-        pip install -v -e .
+        pip install -e .[test] torch==${{ matrix.torch }} --extra-index-url https://download.pytorch.org/whl/cpu
         # Test PPE is importable with minimum dependency
         python -c 'import pytorch_pfn_extras'
 
     - name: Code Style
       run: |
-        pip install pysen black==23.3.0 flake8==4.0.1 isort==5.10.1 mypy==0.991
-        pip install types-PyYAML types-setuptools
-        cp "$(pip show torch | awk '/^Location:/ { print $2 }')/torch/__init__.py" stubs/torch/__init__.py
-        MYPYPATH="${PWD}/stubs" pysen run lint
+        make lint
 
     - name: Code Style (Examples)
       run: |
-        pysen --config ./example/pysen.toml run lint
-
-    - name: Install for pytest
-      run: |
-        pip install matplotlib tensorboard ipython ipywidgets pandas optuna onnx pytorch-ignite
-        pip install -v -e '.[test]' 'torch==${{ matrix.torch }}' --extra-index-url https://download.pytorch.org/whl/cpu
-        pip list -v
-        # Test PPE is importable with pytest dependency
-        python -c 'import pytorch_pfn_extras'
+        make example_lint
 
     - name: Test CPU only
       run: |
-        pytest -m "not gpu" tests
+        make cputest

Makefile

@@ -0,0 +1,36 @@
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+MAKEFLAGS += --warn-undefined-variables
+.DEFAULT_GOAL := help
+
+PWD := $(realpath $(dir $(abspath $(firstword $(MAKEFILE_LIST)))))
+
+PY := python
+PIP := $(PY) -m pip
+
+.PHONY: format
+format: ## Format the Python code.	
+	cp "$$($(PIP) show torch | awk '/^Location:/ { print $$2 }')/torch/__init__.py" stubs/torch/__init__.py
+	trap "rm -f stubs/torch/__init__.py" EXIT; MYPYPATH="$(PWD)/stubs" $(PY) -m pysen run format lint
+
+.PHONY: lint
+lint: ## Lint the Python code.
+	cp "$$($(PIP) show torch | awk '/^Location:/ { print $$2 }')/torch/__init__.py" stubs/torch/__init__.py
+	trap "rm -f stubs/torch/__init__.py" EXIT; MYPYPATH="$(PWD)/stubs" $(PY) -m pysen run lint
+
+.PHONY: test
+test: ## Run all tests.
+	$(PY) -m pytest tests
+
+.PHONY: cputest
+cputest: ## Run all tests except for ones requiring GPU.
+	$(PY) -m pytest -m "not gpu" tests
+
+.PHONY: example_lint
+example_lint: ## Format the Python code.
+	$(PY) -m pysen --config ./example/pysen.toml run lint
+
+.PHONY: help
+help: ## Display this help message.
+	@grep -E '^[%%a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'

example/pysen.toml

@@ -11,6 +11,7 @@ enable_isort = false
 enable_mypy = true
 mypy_preset = "entry"
 line_length = 80
+py_version = "py38"
 
 [[tool.pysen.lint.mypy_targets]]
   paths = ["."]

pytorch_pfn_extras/training/extensions/print_report_notebook.py

@@ -44,7 +44,7 @@ def __init__(
         self._widget = HTML()
 
     def initialize(self, manager: ExtensionsManagerProtocol) -> None:
-        display(self._widget)
+        display(self._widget)  # type: ignore[no-untyped-call]
         super(PrintReportNotebook, self).initialize(manager)
 
     @property

pytorch_pfn_extras/training/extensions/progress_bar_notebook.py

@@ -76,7 +76,7 @@ def initialize(self, manager: ExtensionsManagerProtocol) -> None:
             self._init_status_template()
 
         self.update(manager.iteration, manager.epoch_detail)
-        display(self._widget)
+        display(self._widget)  # type: ignore[no-untyped-call]
 
     def __call__(self, manager: ExtensionsManagerProtocol) -> None:
         length, unit = self._training_length

pytorch_pfn_extras/training/extensions/util.py

@@ -17,8 +17,10 @@
 
 
 def _is_notebook() -> bool:
-    if _ipython_available and get_ipython() is not None:
-        return "IPKernelApp" in get_ipython().config
+    if _ipython_available and get_ipython() is not None:  # type: ignore[no-untyped-call]
+        return (
+            "IPKernelApp" in get_ipython().config  # type: ignore[no-untyped-call]
+        )
     return False
 
 

setup.py

@@ -16,7 +16,27 @@
     license="MIT License",
     install_requires=["numpy", "packaging", "torch", "typing-extensions>=3.10"],
     extras_require={
-        "test": ["pytest", "onnxruntime", "torchvision"],
+        "test": [
+            "pytest",
+            "onnxruntime",
+            "torchvision",
+            "torchaudio",
+            "pysen",
+            "black==23.3.0",
+            "flake8==4.0.1",
+            "isort==5.10.1",
+            "mypy==1.3.0",
+            "types-PyYAML",
+            "types-setuptools",
+            "matplotlib",
+            "tensorboard",
+            "ipython",
+            "ipywidgets",
+            "pandas",
+            "optuna",
+            "onnx",
+            "pytorch-ignite",
+        ],
         "onnx": ["onnx"],
     },
     python_requires=">=3.6.0",