From caf8b3b21bab4e7abc9ef16f9769902c4dc96d6e Mon Sep 17 00:00:00 2001 From: Liang Yam Date: Fri, 3 May 2024 17:11:29 -0400 Subject: [PATCH] HAB-#124 Add support for Python linting to check for issues before packaging --- README.md | 46 +++++++++- .../habushu/ValidatePythonSourceMojo.java | 89 +++++++++++++++++++ .../habushu/ValidatePythonTestMojo.java | 86 ++++++++++++++++++ .../resources/META-INF/plexus/components.xml | 6 +- habushu-mixology-consumer/pyproject.toml | 2 + .../generate-report-with-manual-tests.py | 2 +- .../tests/features/steps/unpack_deps.py | 2 +- habushu-mixology/pyproject.toml | 1 + .../tests/features/steps/reference_src.py | 4 +- .../steps/reference_test_resources.py | 2 +- .../steps/use_environment_variables.py | 2 +- 11 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonSourceMojo.java create mode 100644 habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonTestMojo.java diff --git a/README.md b/README.md index 98079ec..afedb2f 100644 --- a/README.md +++ b/README.md @@ -703,7 +703,51 @@ Default: `true` #### omitSkippedTests #### -Controls whether skipped tests should be completely omitted from test reports rather than showing up as a skip / failure. This mimics the default behavior of Cucumber and will have no effect if outputCuucumberStyleTestReports is not set to `true` +Controls whether skipped tests should be completely omitted from test reports rather than showing up as a skip / failure. This mimics the default behavior of Cucumber and will have no effect if outputCucumberStyleTestReports is not set to `true` + +Default: `true` + +#### lintSource #### + +Controls if linting is enabled for the source module. + +Default: `true` + +#### sourceLintDisabledChecker #### + +Controls the disabled checkers for linting in the source module. By default, checks for Convention (C), Refactor (R), and Warning (W) are disabled. + +Default: `C,R,W` + +#### sourceLintEnabledChecker #### + +Controls the enabled checkers for linting in the source module. + +#### sourceFailOnLintErrors #### + +Controls whether the build will continue if lint identifies code that violate checkers in the source module. + +Default: `true` + +#### lintTest #### + +Controls if linting is enabled for the test module. + +Default: `true` + +#### testLintDisabledChecker #### + +Controls the disabled checkers for linting in the test module. By default, checks for Convention (C), Refactor (R), Warning (W), and function-redefined error (E0102) are disabled. + +Default: `C,R,W,E0102` + +#### testLintEnabledChecker #### + +Controls the enabled checkers for linting in the test module. + +#### testFailOnLintErrors #### + +Controls whether the build will continue if lint identifies code that violate checkers in the test module. Default: `true` diff --git a/habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonSourceMojo.java b/habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonSourceMojo.java new file mode 100644 index 0000000..05c473f --- /dev/null +++ b/habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonSourceMojo.java @@ -0,0 +1,89 @@ +package org.technologybrewery.habushu; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.technologybrewery.habushu.exec.PoetryCommandHelper; + +/** + * Leverages the lint package to validate both source and test Python + * directories using Poetry's run command. + */ +@Mojo(name = "validate-python-source", defaultPhase = LifecyclePhase.PROCESS_CLASSES, requiresDependencyResolution = ResolutionScope.COMPILE) +public class ValidatePythonSourceMojo extends AbstractHabushuMojo { + + protected static final String LINT_PACKAGE = "pylint"; + + /** + * By default, linting will be enabled on the source module. Can be configured to false so that linting is + * not triggered during build. + */ + @Parameter(property = "habushu.lintSource", required = false, defaultValue = "true") + private boolean lintSource; + + /** + * Specifies disabled checkers for lint on source module. + */ + @Parameter(property = "habushu.sourceLintDisabledChecker", required = false, defaultValue = "C,R,W") + private String sourceLintDisabledChecker; + + /** + * Specifies enabled checkers for lint on source module. + */ + @Parameter(property = "habushu.sourceLintEnabledChecker", required = false) + private String sourceLintEnabledChecker; + + /** + * By default, build will stop if lint errors are found in source module. Can be configured to true so that + * build will continue. + */ + @Parameter(property = "habushu.sourceFailOnLintErrors", required = false, defaultValue = "true") + private boolean sourceFailOnLintErrors; + + @Override + public void doExecute() throws MojoExecutionException { + if (lintSource) { + lintSource(); + } + } + + public void lintSource() throws MojoExecutionException { + PoetryCommandHelper poetryHelper = createPoetryCommandHelper(); + List executeLintArgsSource = new ArrayList<>(); + if (this.sourceDirectory.exists()) { + executeLintArgsSource.addAll(Arrays.asList("run", LINT_PACKAGE)); + executeLintArgsSource.add(getCanonicalPathForFile(sourceDirectory)); + + if (!poetryHelper.isDependencyInstalled(LINT_PACKAGE)) { + getLog().info( + String.format("%s dependency not specified in pyproject.toml - installing now...", LINT_PACKAGE)); + poetryHelper.installDevelopmentDependency(LINT_PACKAGE); + } + + if (StringUtils.isNotEmpty(sourceLintDisabledChecker)) { + executeLintArgsSource.addAll(Arrays.asList("--disable", sourceLintDisabledChecker)); + } + + if (StringUtils.isNotEmpty(sourceLintEnabledChecker)) { + executeLintArgsSource.addAll(Arrays.asList("--enable", sourceLintEnabledChecker)); + } + + if (!sourceFailOnLintErrors) { + executeLintArgsSource.add("--exit-zero"); + } + + getLog().info("Validating code in source directory using Pylint..."); + poetryHelper.executeAndLogOutput(executeLintArgsSource); + } else { + getLog().warn(String.format("Configured source (%s) directory does not exist - skipping...", + sourceDirectory)); + } + } +} diff --git a/habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonTestMojo.java b/habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonTestMojo.java new file mode 100644 index 0000000..a1c0ab5 --- /dev/null +++ b/habushu-maven-plugin/src/main/java/org/technologybrewery/habushu/ValidatePythonTestMojo.java @@ -0,0 +1,86 @@ +package org.technologybrewery.habushu; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.technologybrewery.habushu.exec.PoetryCommandHelper; + +@Mojo(name = "validate-python-test", defaultPhase = LifecyclePhase.PROCESS_TEST_CLASSES, requiresDependencyResolution = ResolutionScope.COMPILE) +public class ValidatePythonTestMojo extends AbstractHabushuMojo { + + protected static final String LINT_PACKAGE = "pylint"; + + /** + * By default, linting will be enabled on the test module. Can be configured to false so that linting is + * not triggered during build. + */ + @Parameter(property = "habushu.lintTest", required = false, defaultValue = "true") + private boolean lintTest; + + /** + * Specifies disabled checkers for lint on test module. + */ + @Parameter(property = "habushu.testLintDisabledChecker", required = false, defaultValue = "C,R,W,E0102") + private String testLintDisabledChecker; + + /** + * Specifies enabled checkers for lint on test module. + */ + @Parameter(property = "habushu.testLintEnabledChecker", required = false) + private String testLintEnabledChecker; + + /** + * By default, build will stop if lint errors are found in test module. Can be configured to true so that + * pylint does interrupt build. + */ + @Parameter(property = "habushu.testFailOnLintErrors", required = false, defaultValue = "true") + private boolean testFailOnLintErrors; + + @Override + public void doExecute() throws MojoExecutionException { + if (lintTest) { + lintTest(); + } + } + + public void lintTest() throws MojoExecutionException { + PoetryCommandHelper poetryHelper = createPoetryCommandHelper(); + List executeLintArgsTest = new ArrayList<>(); + if (this.testDirectory.exists()) { + executeLintArgsTest.addAll(Arrays.asList("run", LINT_PACKAGE)); + executeLintArgsTest.add(getCanonicalPathForFile(testDirectory)); + if (!poetryHelper.isDependencyInstalled(LINT_PACKAGE)) { + getLog().info( + String.format("%s dependency not specified in pyproject.toml - installing now...", LINT_PACKAGE)); + poetryHelper.installDevelopmentDependency(LINT_PACKAGE); + } + + if (StringUtils.isNotEmpty(testLintDisabledChecker)) { + executeLintArgsTest.addAll(Arrays.asList("--disable", testLintDisabledChecker)); + } + + if (StringUtils.isNotEmpty(testLintEnabledChecker)) { + executeLintArgsTest.addAll(Arrays.asList("--enable", testLintEnabledChecker)); + } + + if (!testFailOnLintErrors) { + executeLintArgsTest.add("--exit-zero"); + } + + executeLintArgsTest.add("--recursive=true"); + + getLog().info("Validating code in test directory using Pylint..."); + poetryHelper.executeAndLogOutput(executeLintArgsTest); + } else { + getLog().warn(String.format("Configured test (%s) directory does not exist - skipping...", + testDirectory)); + } + } +} diff --git a/habushu-maven-plugin/src/main/resources/META-INF/plexus/components.xml b/habushu-maven-plugin/src/main/resources/META-INF/plexus/components.xml index 0bac4cf..0c59194 100644 --- a/habushu-maven-plugin/src/main/resources/META-INF/plexus/components.xml +++ b/habushu-maven-plugin/src/main/resources/META-INF/plexus/components.xml @@ -22,7 +22,11 @@ org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:retrieve-wheels org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:install-dependencies - org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:format-python + + org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:format-python, + org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:validate-python-source + + org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:validate-python-test org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:behave-bdd-test org.technologybrewery.habushu:habushu-maven-plugin:${project.version}:build-deployment-artifacts diff --git a/habushu-mixology-consumer/pyproject.toml b/habushu-mixology-consumer/pyproject.toml index 8229197..8d0a85c 100644 --- a/habushu-mixology-consumer/pyproject.toml +++ b/habushu-mixology-consumer/pyproject.toml @@ -13,6 +13,8 @@ python = "^3.11" [tool.poetry.group.dev.dependencies] kappa-maki = ">=1.0.0" +behave-cucumber-formatter = ">=1.0.1" +pylint = ">=3.1.0" [tool.poetry.dev-dependencies] black = ">=24.3.0" diff --git a/habushu-mixology-consumer/tests/features/steps/generate-report-with-manual-tests.py b/habushu-mixology-consumer/tests/features/steps/generate-report-with-manual-tests.py index 488514e..1635952 100644 --- a/habushu-mixology-consumer/tests/features/steps/generate-report-with-manual-tests.py +++ b/habushu-mixology-consumer/tests/features/steps/generate-report-with-manual-tests.py @@ -1,4 +1,4 @@ -from behave import * +from behave import given, when, then # pylint: disable=no-name-in-module @given("a Python module with at least one automated test and at least one manual test") diff --git a/habushu-mixology-consumer/tests/features/steps/unpack_deps.py b/habushu-mixology-consumer/tests/features/steps/unpack_deps.py index f608dd9..350d4f9 100644 --- a/habushu-mixology-consumer/tests/features/steps/unpack_deps.py +++ b/habushu-mixology-consumer/tests/features/steps/unpack_deps.py @@ -1,4 +1,4 @@ -from behave import * +from behave import given, when, then # pylint: disable=no-name-in-module from krausening.properties import PropertyManager from habushu_mixology_consumer.nested.simple_dependency import ( call_worker_from_nested_dir, diff --git a/habushu-mixology/pyproject.toml b/habushu-mixology/pyproject.toml index 9d178e1..e98e6f0 100644 --- a/habushu-mixology/pyproject.toml +++ b/habushu-mixology/pyproject.toml @@ -16,6 +16,7 @@ black = ">=24.3.0" behave = ">=1.2.6" grpcio-tools = "^1.48.0" python-dotenv = "^0.20.0" +pylint = ">=3.1.0" [tool.poetry.group.dev.dependencies] kappa-maki = ">=1.0.0" diff --git a/habushu-mixology/tests/features/steps/reference_src.py b/habushu-mixology/tests/features/steps/reference_src.py index f8df9a2..51ecf8c 100644 --- a/habushu-mixology/tests/features/steps/reference_src.py +++ b/habushu-mixology/tests/features/steps/reference_src.py @@ -1,4 +1,4 @@ -from behave import * +from behave import when, then # pylint: disable=no-name-in-module from habushu_mixology.reusable_module.worker import SubWorker from habushu_mixology.helloworld import generate_random_string from habushu_mixology.generated import person_pb2 @@ -9,7 +9,7 @@ def step_impl(context): logging.info("Referencing a src file...") context.random = generate_random_string(5) - person = person_pb2.Person() + person = person_pb2.Person() # pylint: disable=no-member person.email = "habushu@gmail.com" context.person = person diff --git a/habushu-mixology/tests/features/steps/reference_test_resources.py b/habushu-mixology/tests/features/steps/reference_test_resources.py index 8b40625..c8db419 100644 --- a/habushu-mixology/tests/features/steps/reference_test_resources.py +++ b/habushu-mixology/tests/features/steps/reference_test_resources.py @@ -1,4 +1,4 @@ -from behave import * +from behave import when, then # pylint: disable=no-name-in-module @when("I reference a test resource in my test file") diff --git a/habushu-mixology/tests/features/steps/use_environment_variables.py b/habushu-mixology/tests/features/steps/use_environment_variables.py index 17a4937..22fd62d 100644 --- a/habushu-mixology/tests/features/steps/use_environment_variables.py +++ b/habushu-mixology/tests/features/steps/use_environment_variables.py @@ -1,4 +1,4 @@ -from behave import * +from behave import when, then # pylint: disable=no-name-in-module from test_config import TestConfig