Local plugin support/documentation

oxsecurity · Jun 3, 2022 · 23f4951 · 23f4951
1 parent 100a4d9
commit 23f4951
Show file tree

Hide file tree

Showing 8 changed files with 11,923 additions and 22,250 deletions.
diff --git a/.automation/generated/flavors-stats.json b/.automation/generated/flavors-stats.json
diff --git a/.automation/generated/linter-licenses.json b/.automation/generated/linter-licenses.json
@@ -4,7 +4,7 @@
     "bandit": "Apache-2.0",
     "black": "MIT",
     "checkov": "Apache-2.0",
-    "checkstyle": "Other",
+    "checkstyle": "LGPL-2.1",
     "coffeelint": "Other",
     "cspell": "MIT",
     "dartanalyzer": "BSD-3-Clause",

diff --git a/.automation/generated/megalinter-users.json b/.automation/generated/megalinter-users.json
diff --git a/.mega-linter.yml b/.mega-linter.yml
@@ -33,6 +33,7 @@ JSON_REPORTER: true
 GITHUB_STATUS_REPORTER: false
 PLUGINS:
   - https://raw.githubusercontent.com/megalinter/megalinter/main/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml
+  - file://.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml
 PRE_COMMANDS:
   - command: echo "This is MegaLinter PRE_COMMAND on own MegaLinter ! :)"
     cwd: "root"

diff --git a/docs/all_linters.md b/docs/all_linters.md
@@ -27,7 +27,7 @@
 | [**editorconfig-checker**](https://github.com/editorconfig-checker/editorconfig-checker){target=_blank} |     2.4.0      |   [MIT](licenses/editorconfig-checker.md)   | [EDITORCONFIG](descriptors/editorconfig_editorconfig_checker.md)                                                                                                                           |       :heart:       |                   [MegaLinter reference](https://github.com/editorconfig-checker/editorconfig-checker#mega-linter){target=_blank}                   |
 | [**eslint**](https://github.com/eslint/eslint){target=_blank}                                           |     8.16.0     |          [MIT](licenses/eslint.md)          | [JAVASCRIPT](descriptors/javascript_eslint.md)<br/> [JSX](descriptors/jsx_eslint.md)<br/> [TSX](descriptors/tsx_eslint.md)<br/> [TYPESCRIPT](descriptors/typescript_eslint.md)             |       :heart:       |                        [MegaLinter reference](https://eslint.org/docs/user-guide/integrations#source-control){target=_blank}                        |
 | [**eslint-plugin-jsonc**](https://github.com/ota-meshi/eslint-plugin-jsonc){target=_blank}              |     2.3.0      |   [MIT](licenses/eslint-plugin-jsonc.md)    | [JSON](descriptors/json_eslint_plugin_jsonc.md)                                                                                                                                            |       :heart:       |                        [MegaLinter reference](https://eslint.org/docs/user-guide/integrations#source-control){target=_blank}                        |
-| [**flake8**](https://github.com/PyCQA/flake8){target=_blank}                                            |     4.0.1      |          [MIT](licenses/flake8.md)          | [PYTHON](descriptors/python_flake8.md)                                                                                                                                                     |   :white_circle:    |                                            [Repository](https://github.com/PyCQA/flake8){target=_blank}                                             |
+| [**flake8**](https://github.com/PyCQA/flake8){target=_blank}                                            |     4.0.1      |         [Other](licenses/flake8.md)         | [PYTHON](descriptors/python_flake8.md)                                                                                                                                                     |   :white_circle:    |                                            [Repository](https://github.com/PyCQA/flake8){target=_blank}                                             |
 | [**gherkin-lint**](https://github.com/vsiakka/gherkin-lint){target=_blank}                              |      N/A       |                  <!-- -->                   | [GHERKIN](descriptors/gherkin_gherkin_lint.md)                                                                                                                                             |   :white_circle:    |                                         [Web Site](https://github.com/vsiakka/gherkin-lint){target=_blank}                                          |
 | [**git_diff**](https://github.com/git/git){target=_blank}                                               |     2.30.3     |        [Other](licenses/git_diff.md)        | [GIT](descriptors/git_git_diff.md)                                                                                                                                                         |      <!-- -->       |                                               [Repository](https://github.com/git/git){target=_blank}                                               |
 | [**golangci-lint**](https://github.com/golangci/golangci-lint){target=_blank}                           |     1.46.2     |    [GPL-3.0](licenses/golangci-lint.md)     | [GO](descriptors/go_golangci_lint.md)                                                                                                                                                      |   :white_circle:    |                                       [Repository](https://github.com/golangci/golangci-lint){target=_blank}                                        |

diff --git a/docs/all_users.md b/docs/all_users.md
@@ -62,9 +62,9 @@
 [![cbg-ethz/SARS-CoV-2_Analysis - GitHub](https://gh-card.dev/repos/cbg-ethz/SARS-CoV-2_Analysis.svg?fullname=)](https://github.com/cbg-ethz/SARS-CoV-2_Analysis){target=_blank}
 [![mundialis/actinia-stac-plugin - GitHub](https://gh-card.dev/repos/mundialis/actinia-stac-plugin.svg?fullname=)](https://github.com/mundialis/actinia-stac-plugin){target=_blank}
 [![r-spacex/submanager - GitHub](https://gh-card.dev/repos/r-spacex/submanager.svg?fullname=)](https://github.com/r-spacex/submanager){target=_blank}
-[![bsrodrigs/terraform-aws-fully-connected-vpn - GitHub](https://gh-card.dev/repos/bsrodrigs/terraform-aws-fully-connected-vpn.svg?fullname=)](https://github.com/bsrodrigs/terraform-aws-fully-connected-vpn){target=_blank}
+[![bsrodrigs/terraform-aws-selfconfig-cgw - GitHub](https://gh-card.dev/repos/bsrodrigs/terraform-aws-selfconfig-cgw.svg?fullname=)](https://github.com/bsrodrigs/terraform-aws-selfconfig-cgw){target=_blank}
 [![MTUCI-VR/shooter-project - GitHub](https://gh-card.dev/repos/MTUCI-VR/shooter-project.svg?fullname=)](https://github.com/MTUCI-VR/shooter-project){target=_blank}
-[![lpmatos/docker-crypto-miner - GitHub](https://gh-card.dev/repos/lpmatos/docker-crypto-miner.svg?fullname=)](https://github.com/lpmatos/docker-crypto-miner){target=_blank}
+[![ci-monk/docker-crypto-miner - GitHub](https://gh-card.dev/repos/ci-monk/docker-crypto-miner.svg?fullname=)](https://github.com/ci-monk/docker-crypto-miner){target=_blank}
 [![tonic-team/tonic.site - GitHub](https://gh-card.dev/repos/tonic-team/tonic.site.svg?fullname=)](https://github.com/tonic-team/tonic.site){target=_blank}
 [![epleypa/Home-AssistantConfig - GitHub](https://gh-card.dev/repos/epleypa/Home-AssistantConfig.svg?fullname=)](https://github.com/epleypa/Home-AssistantConfig){target=_blank}
 [![ewencluley/final-curtain - GitHub](https://gh-card.dev/repos/ewencluley/final-curtain.svg?fullname=)](https://github.com/ewencluley/final-curtain){target=_blank}

diff --git a/docs/plugins.md b/docs/plugins.md
@@ -10,14 +10,17 @@ But any linter can be callable within MegaLinter thanks to the plugin mechanism
 
 ## Use plugins
 
-Just add plugin URLs in `PLUGINS` property of `.mega-linter.yml`
+Add plugin URLs in `PLUGINS` property of `.mega-linter.yml`. URLs must either begin with "https://" or take the form of "file://\<path\>", where \<path\> points to a valid plugin descriptor file.
+
+> Note: Both \<path\> and the default mount directory (/tmp/lint/\<path\>) will be checked for a valid descriptor.
 
 ### Example
 
 ```yaml
 PLUGINS:
   - https://raw.githubusercontent.com/megalinter/megalinter/main/.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml
   - https://raw.githubusercontent.com/cookiejar/mega-linter-plugin-cookietemple/main/cookietemple.megalinter-descriptor.yml
+  - file://.automation/test/mega-linter-plugin-test/test.megalinter-descriptor.yml
 ```
 
 ## Plugins Catalog
@@ -32,9 +35,10 @@ Submit a PR if you want your plugin to appear here :)
 
 You can implement your own descriptors and load them as plugins during MegaLinter runtime
 
-- Plugins descriptor files must be named **\*\*.megalinter-descriptor.yml** and respect [MegaLinter Json Schema](https://github.com/megalinter/megalinter/blob/main/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json)
 - Descriptor format is exactly the same than [MegaLinter embedded ones](https://github.com/megalinter/megalinter/tree/main/megalinter/descriptors) ([see json schema documentation](https://megalinter.github.io/json-schemas/descriptor.html))
+- Plugins descriptor files must be named **\*\*.megalinter-descriptor.yml** and respect [MegaLinter Json Schema](https://github.com/megalinter/megalinter/blob/main/megalinter/descriptors/schemas/megalinter-descriptor.jsonschema.json)
 - Plugins must be hosted in a url containing **\*\*/mega-linter-plugin-\*\*/**
+- File URLs must conform to the same directory and file naming criteria as defined above.
 
 ### Limitations
 

diff --git a/megalinter/plugin_factory.py b/megalinter/plugin_factory.py
@@ -3,6 +3,7 @@
 import shutil
 import subprocess
 import sys
+import os
 
 import requests
 import yaml
@@ -24,8 +25,9 @@ def initialize_plugins():
 
 # Load plugin descriptor
 def load_plugin(plugin):
-    if plugin.startswith("https://"):
-        # Check validity of plugin URL
+    # Check if plugin is a URL or local path
+    if plugin.startswith("https://") or plugin.startswith("file://"):
+        # Check validity of plugin URL/path
         descriptor_file = "/megalinter-descriptors/" + plugin.rsplit("/", 1)[1]
         if "/mega-linter-plugin-" not in plugin:
             raise Exception(
@@ -35,23 +37,38 @@ def load_plugin(plugin):
             raise Exception(
                 "[Plugins] Plugin descriptor file must end with .megalinter-descriptor.yml"
             )
+
         # Download plugin and write it in megalinter
         try:
-            r = requests.get(plugin, allow_redirects=True)
-            plugin_descriptor = yaml.safe_load(r.content)
+            if plugin.startswith("https://"):
+                r = requests.get(plugin, allow_redirects=True).content
+            else:
+                # From file://<path>, test both <path> and /tmp/lint/<path>
+                plugin_path = plugin.split("file://")[1]
+                if not os.path.isfile(plugin_path):
+                    plugin_path = "/tmp/lint/" + plugin_path
+                    if not os.path.isfile(plugin_path):
+                        raise Exception(f"[Plugins] Local plugin descriptor {plugin} not found")
+                # Make sure plugin file is readable and not empty
+                if not os.access(plugin_path, os.R_OK):
+                    raise Exception(f"[Plugins] Local plugin descriptor {plugin} not readable")
+                if os.stat(plugin_path).st_size == 0:
+                    raise Exception(f"[Plugins] Plugin descriptor {plugin} is empty")
+                r = open(plugin_path, "r").read()
+            plugin_descriptor = yaml.safe_load(r)
             plugin_descriptor["is_plugin"] = True
             with open(descriptor_file, "w") as outfile:
                 yaml.dump(plugin_descriptor, outfile)
             logging.info(
                 f"[Plugins] Loaded plugin descriptor {descriptor_file} from {plugin}"
             )
         except Exception as e:
-            raise Exception(f"[Plugins] Unable to load plugin {plugin}:\n{str(e)}")
+            raise Exception(f"[Plugins] Unable to load remote plugin {plugin}:\n{str(e)}")
         return descriptor_file
     else:
         raise Exception(
             "[Plugins] Plugin descriptors must follow the format"
-            f" https://**/mega-linter-plugin-**/**.mega-linter-descriptor.yml (wrong value {plugin})"
+            f" https://**/mega-linter-plugin-**/**.mega-linter-descriptor.yml or file://**/mega-linter-plugin-**/**.mega-linter-descriptor.yml (wrong value {plugin})"
         )