diff --git a/.automation/generated/flavors-stats.json b/.automation/generated/flavors-stats.json index 954d5da9931..62fc2d5bca5 100644 --- a/.automation/generated/flavors-stats.json +++ b/.automation/generated/flavors-stats.json @@ -13903,4 +13903,4 @@ 73005 ] ] -} \ No newline at end of file +} diff --git a/.automation/generated/linter-licenses.json b/.automation/generated/linter-licenses.json index 4c5837ac48a..a401f74349f 100644 --- a/.automation/generated/linter-licenses.json +++ b/.automation/generated/linter-licenses.json @@ -4,7 +4,7 @@ "bandit": "Apache-2.0", "black": "MIT", "checkov": "Apache-2.0", - "checkstyle": "Other", + "checkstyle": "LGPL-2.1", "coffeelint": "Other", "cspell": "MIT", "dartanalyzer": "BSD-3-Clause", diff --git a/.automation/test/mega-linter-plugin-test/test-empty.megalinter-descriptor.yml b/.automation/test/mega-linter-plugin-test/test-empty.megalinter-descriptor.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml b/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml index 9460f20e035..c3f838a4a5e 100644 --- a/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml +++ b/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml @@ -1,13 +1,12 @@ descriptor_id: TEST descriptor_type: format file_extensions: - - ".jsonc" + - .jsonc linters: - # JSONLINT - - linter_name: jsonlint - linter_url: https://github.com/zaach/jsonlint - examples: - - "jsonlint myfile.jsonc" + - examples: + - jsonlint myfile.jsonc install: dockerfile: - - RUN npm install jsonlint -g && echo "This is my test MegaLinter plugin !" + - RUN npm install jsonlint -g && echo "This is my test MegaLinter plugin!" + linter_name: jsonlint + linter_url: https://github.com/zaach/jsonlint diff --git a/.jscpd.json b/.jscpd.json index 93a9658961d..7a700646876 100644 --- a/.jscpd.json +++ b/.jscpd.json @@ -22,6 +22,7 @@ "**/megalinter/linters/RakuLinter.py", "**/megalinter/reporters/*.py", "**/megalinter/tests/test_megalinter/mega_linter*", + "**/megalinter/tests/test_megalinter/plugins_test.py*", "**/megalinter/tests/test_megalinter/helpers/utilstest.py", "**/*.test.js", "**/CHANGELOG.md", diff --git a/README.md b/README.md index a3a5a53323c..333c280a4f4 100644 --- a/README.md +++ b/README.md @@ -891,7 +891,9 @@ But any linter can be callable within MegaLinter thanks to the plugin mechanism ### Use plugins -Just add plugin URLs in `PLUGINS` property of `.mega-linter.yml` +Add plugin URLs in `PLUGINS` property of `.mega-linter.yml`. URLs must either begin with "https://" or take the form of "file://\", where \ points to a valid plugin descriptor file. + +> Note: Both \ and the default mount directory (/tmp/lint/\) will be checked for a valid descriptor. #### Example @@ -899,6 +901,7 @@ Just add plugin URLs in `PLUGINS` property of `.mega-linter.yml` PLUGINS: - https://raw.githubusercontent.com/megalinter/megalinter/main/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml - https://raw.githubusercontent.com/cookiejar/mega-linter-plugin-cookietemple/main/cookietemple.megalinter-descriptor.yml + - file://.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml ``` ### Plugins Catalog @@ -913,13 +916,15 @@ Submit a PR if you want your plugin to appear here :) You can implement your own descriptors and load them as plugins during MegaLinter runtime -- Plugins descriptor files must be named **\*\*.megalinter-descriptor.yml** and respect [MegaLinter Json Schema](https://github.com/megalinter/megalinter/blob/main/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json) - Descriptor format is exactly the same than [MegaLinter embedded ones](https://github.com/megalinter/megalinter/tree/main/megalinter/descriptors) ([see json schema documentation](https://megalinter.github.io/json-schemas/descriptor.html)) +- Plugins descriptor files must be named **\*\*.megalinter-descriptor.yml** and respect [MegaLinter Json Schema](https://github.com/megalinter/megalinter/blob/main/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json) - Plugins must be hosted in a url containing **\*\*/mega-linter-plugin-\*\*/** +- File URLs must conform to the same directory and file naming criteria as defined above. #### Limitations - For now, the only `install` attributes managed are `dockerfile` instructions starting by `RUN` + diff --git a/docs/all_linters.md b/docs/all_linters.md index 5eb45e8700e..6a972dc865f 100644 --- a/docs/all_linters.md +++ b/docs/all_linters.md @@ -27,7 +27,7 @@ | [**editorconfig-checker**](https://github.com/editorconfig-checker/editorconfig-checker){target=_blank} | 2.4.0 | [MIT](licenses/editorconfig-checker.md) | [EDITORCONFIG](descriptors/editorconfig_editorconfig_checker.md) | :heart: | [MegaLinter reference](https://github.com/editorconfig-checker/editorconfig-checker#mega-linter){target=_blank} | | [**eslint**](https://github.com/eslint/eslint){target=_blank} | 8.17.0 | [MIT](licenses/eslint.md) | [JAVASCRIPT](descriptors/javascript_eslint.md)
[JSX](descriptors/jsx_eslint.md)
[TSX](descriptors/tsx_eslint.md)
[TYPESCRIPT](descriptors/typescript_eslint.md) | :heart: | [MegaLinter reference](https://eslint.org/docs/user-guide/integrations#source-control){target=_blank} | | [**eslint-plugin-jsonc**](https://github.com/ota-meshi/eslint-plugin-jsonc){target=_blank} | 2.3.0 | [MIT](licenses/eslint-plugin-jsonc.md) | [JSON](descriptors/json_eslint_plugin_jsonc.md) | :heart: | [MegaLinter reference](https://eslint.org/docs/user-guide/integrations#source-control){target=_blank} | -| [**flake8**](https://github.com/PyCQA/flake8){target=_blank} | 4.0.1 | [MIT](licenses/flake8.md) | [PYTHON](descriptors/python_flake8.md) | :white_circle: | [Repository](https://github.com/PyCQA/flake8){target=_blank} | +| [**flake8**](https://github.com/PyCQA/flake8){target=_blank} | 4.0.1 | [Other](licenses/flake8.md) | [PYTHON](descriptors/python_flake8.md) | :white_circle: | [Repository](https://github.com/PyCQA/flake8){target=_blank} | | [**gherkin-lint**](https://github.com/vsiakka/gherkin-lint){target=_blank} | N/A | | [GHERKIN](descriptors/gherkin_gherkin_lint.md) | :white_circle: | [Web Site](https://github.com/vsiakka/gherkin-lint){target=_blank} | | [**git_diff**](https://github.com/git/git){target=_blank} | 2.30.3 | [Other](licenses/git_diff.md) | [GIT](descriptors/git_git_diff.md) | | [Repository](https://github.com/git/git){target=_blank} | | [**golangci-lint**](https://github.com/golangci/golangci-lint){target=_blank} | 1.46.2 | [GPL-3.0](licenses/golangci-lint.md) | [GO](descriptors/go_golangci_lint.md) | :white_circle: | [Repository](https://github.com/golangci/golangci-lint){target=_blank} | diff --git a/docs/all_users.md b/docs/all_users.md index 6e74e7f1d1f..6f122299c11 100644 --- a/docs/all_users.md +++ b/docs/all_users.md @@ -65,9 +65,9 @@ [![cbg-ethz/SARS-CoV-2_Analysis - GitHub](https://gh-card.dev/repos/cbg-ethz/SARS-CoV-2_Analysis.svg?fullname=)](https://github.com/cbg-ethz/SARS-CoV-2_Analysis){target=_blank} [![mundialis/actinia-stac-plugin - GitHub](https://gh-card.dev/repos/mundialis/actinia-stac-plugin.svg?fullname=)](https://github.com/mundialis/actinia-stac-plugin){target=_blank} [![r-spacex/submanager - GitHub](https://gh-card.dev/repos/r-spacex/submanager.svg?fullname=)](https://github.com/r-spacex/submanager){target=_blank} -[![bsrodrigs/terraform-aws-fully-connected-vpn - GitHub](https://gh-card.dev/repos/bsrodrigs/terraform-aws-fully-connected-vpn.svg?fullname=)](https://github.com/bsrodrigs/terraform-aws-fully-connected-vpn){target=_blank} +[![bsrodrigs/terraform-aws-selfconfig-cgw - GitHub](https://gh-card.dev/repos/bsrodrigs/terraform-aws-selfconfig-cgw.svg?fullname=)](https://github.com/bsrodrigs/terraform-aws-selfconfig-cgw){target=_blank} [![MTUCI-VR/shooter-project - GitHub](https://gh-card.dev/repos/MTUCI-VR/shooter-project.svg?fullname=)](https://github.com/MTUCI-VR/shooter-project){target=_blank} -[![lpmatos/docker-crypto-miner - GitHub](https://gh-card.dev/repos/lpmatos/docker-crypto-miner.svg?fullname=)](https://github.com/lpmatos/docker-crypto-miner){target=_blank} +[![ci-monk/docker-crypto-miner - GitHub](https://gh-card.dev/repos/ci-monk/docker-crypto-miner.svg?fullname=)](https://github.com/ci-monk/docker-crypto-miner){target=_blank} [![tonic-team/tonic.site - GitHub](https://gh-card.dev/repos/tonic-team/tonic.site.svg?fullname=)](https://github.com/tonic-team/tonic.site){target=_blank} [![epleypa/Home-AssistantConfig - GitHub](https://gh-card.dev/repos/epleypa/Home-AssistantConfig.svg?fullname=)](https://github.com/epleypa/Home-AssistantConfig){target=_blank} [![ewencluley/final-curtain - GitHub](https://gh-card.dev/repos/ewencluley/final-curtain.svg?fullname=)](https://github.com/ewencluley/final-curtain){target=_blank} diff --git a/docs/plugins.md b/docs/plugins.md index 762f15d6835..ed005e8f86d 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -10,7 +10,9 @@ But any linter can be callable within MegaLinter thanks to the plugin mechanism ## Use plugins -Just add plugin URLs in `PLUGINS` property of `.mega-linter.yml` +Add plugin URLs in `PLUGINS` property of `.mega-linter.yml`. URLs must either begin with "https://" or take the form of "file://\", where \ points to a valid plugin descriptor file. + +> Note: Both \ and the default mount directory (/tmp/lint/\) will be checked for a valid descriptor. ### Example @@ -18,6 +20,7 @@ Just add plugin URLs in `PLUGINS` property of `.mega-linter.yml` PLUGINS: - https://raw.githubusercontent.com/megalinter/megalinter/main/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml - https://raw.githubusercontent.com/cookiejar/mega-linter-plugin-cookietemple/main/cookietemple.megalinter-descriptor.yml + - file://.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml ``` ## Plugins Catalog @@ -32,9 +35,10 @@ Submit a PR if you want your plugin to appear here :) You can implement your own descriptors and load them as plugins during MegaLinter runtime -- Plugins descriptor files must be named **\*\*.megalinter-descriptor.yml** and respect [MegaLinter Json Schema](https://github.com/megalinter/megalinter/blob/main/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json) - Descriptor format is exactly the same than [MegaLinter embedded ones](https://github.com/megalinter/megalinter/tree/main/megalinter/descriptors) ([see json schema documentation](https://megalinter.github.io/json-schemas/descriptor.html)) +- Plugins descriptor files must be named **\*\*.megalinter-descriptor.yml** and respect [MegaLinter Json Schema](https://github.com/megalinter/megalinter/blob/main/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json) - Plugins must be hosted in a url containing **\*\*/mega-linter-plugin-\*\*/** +- File URLs must conform to the same directory and file naming criteria as defined above. ### Limitations diff --git a/megalinter/plugin_factory.py b/megalinter/plugin_factory.py index 80f1abcbe5c..234765a6ffd 100644 --- a/megalinter/plugin_factory.py +++ b/megalinter/plugin_factory.py @@ -1,5 +1,6 @@ # Class to manage MegaLinter plugins import logging +import os import shutil import subprocess import sys @@ -24,8 +25,9 @@ def initialize_plugins(): # Load plugin descriptor def load_plugin(plugin): - if plugin.startswith("https://"): - # Check validity of plugin URL + # Check if plugin is a URL or local path + if plugin.startswith("https://") or plugin.startswith("file://"): + # Check validity of plugin URL/path descriptor_file = "/megalinter-descriptors/" + plugin.rsplit("/", 1)[1] if "/mega-linter-plugin-" not in plugin: raise Exception( @@ -35,10 +37,24 @@ def load_plugin(plugin): raise Exception( "[Plugins] Plugin descriptor file must end with .megalinter-descriptor.yml" ) + # Download plugin and write it in megalinter try: - r = requests.get(plugin, allow_redirects=True) - plugin_descriptor = yaml.safe_load(r.content) + if plugin.startswith("https://"): + r = requests.get(plugin, allow_redirects=True).content + else: + # From file://, test both and /tmp/lint/ + plugin_path = plugin.split("file://")[1] + if not os.access(plugin_path, os.R_OK): + plugin_path = "/tmp/lint/" + plugin_path + if not os.access(plugin_path, os.R_OK): + raise Exception( + f"[Plugins] Local plugin descriptor not found or not readable {plugin}" + ) + if os.stat(plugin_path).st_size == 0: + raise Exception(f"[Plugins] Plugin descriptor is empty: {plugin}") + r = open(plugin_path, "r").read() + plugin_descriptor = yaml.safe_load(r) plugin_descriptor["is_plugin"] = True with open(descriptor_file, "w") as outfile: yaml.dump(plugin_descriptor, outfile) @@ -46,12 +62,15 @@ def load_plugin(plugin): f"[Plugins] Loaded plugin descriptor {descriptor_file} from {plugin}" ) except Exception as e: - raise Exception(f"[Plugins] Unable to load plugin {plugin}:\n{str(e)}") + raise Exception( + f"[Plugins] Unable to load remote plugin {plugin}:\n{str(e)}" + ) return descriptor_file else: raise Exception( "[Plugins] Plugin descriptors must follow the format" - f" https://**/mega-linter-plugin-**/**.mega-linter-descriptor.yml (wrong value {plugin})" + " https://**/mega-linter-plugin-**/**.mega-linter-descriptor.yml or" + f" file://**/mega-linter-plugin-**/**.mega-linter-descriptor.yml (wrong value {plugin})" ) diff --git a/megalinter/tests/test_megalinter/plugins_test.py b/megalinter/tests/test_megalinter/plugins_test.py index 0633706cf2a..6a1415ecbcf 100644 --- a/megalinter/tests/test_megalinter/plugins_test.py +++ b/megalinter/tests/test_megalinter/plugins_test.py @@ -6,7 +6,6 @@ import os import unittest -from git import Repo from megalinter.constants import ML_REPO from megalinter.tests.test_megalinter.helpers import utilstest @@ -20,11 +19,11 @@ def setUp(self): ) def test_load_plugin_success(self): - try: - local_repo = Repo(search_parent_directories=True) - local_branch = local_repo.active_branch.name - except: # noqa: E722 - local_branch = "master" + # {ML_REPO}/local_branch won't necessarily be valid on fork branches. + # Temporary workaround: use megalinter/main branch to host the file. + # It's not a huge issue here because in theory this file will always be available. + # TODO: don't just use the local branch name, but parse {ML_REPO} as well. + local_branch = "main" mega_linter, output = utilstest.call_mega_linter( { "PLUGINS": f"https://raw.githubusercontent.com/{ML_REPO}/" @@ -42,6 +41,48 @@ def test_load_plugin_success(self): self.assertIn("[Plugins] Loaded plugin descriptor", output) self.assertIn("[Plugins] Successful initialization of TEST", output) + def test_load_local_plugin_success(self): + mega_linter, output = utilstest.call_mega_linter( + { + "PLUGINS": "file://.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml", + "LOG_LEVEL": "DEBUG", + "MULTI_STATUS": "false", + "GITHUB_COMMENT_REPORTER": "false", + } + ) + self.assertTrue( + len(mega_linter.linters) > 0, "Linters have been created and run" + ) + self.assertIn("### Processed [TEST] files", output) + self.assertIn("[Plugins] Loaded plugin descriptor", output) + self.assertIn("[Plugins] Successful initialization of TEST", output) + + def test_load_local_plugin_fail(self): + try: + utilstest.call_mega_linter( + { + "PLUGINS": "file://.automation/test/mega-linter-plugin-test/test-fake.megalinter-descriptor.yml", + "LOG_LEVEL": "DEBUG", + "MULTI_STATUS": "false", + "GITHUB_COMMENT_REPORTER": "false", + } + ) + except Exception as e: + self.assertIn("[Plugins] Local plugin descriptor not found or not readable", str(e)) + + def test_load_local_plugin_read_fail(self): + try: + utilstest.call_mega_linter( + { + "PLUGINS": "file://.automation/test/mega-linter-plugin-test/test-empty.megalinter-descriptor.yml", + "LOG_LEVEL": "DEBUG", + "MULTI_STATUS": "false", + "GITHUB_COMMENT_REPORTER": "false", + } + ) + except Exception as e: + self.assertIn("[Plugins] Plugin descriptor is empty:", str(e)) + def test_load_plugin_http_error(self): try: utilstest.call_mega_linter( @@ -54,7 +95,7 @@ def test_load_plugin_http_error(self): } ) except Exception as e: - self.assertIn("[Plugins] Unable to load plugin", str(e)) + self.assertIn("[Plugins] Unable to load remote plugin", str(e)) def test_load_plugin_host_url_error_1(self): try: