From 1e261f60fc611555f7d66384099671b08fc73555 Mon Sep 17 00:00:00 2001 From: Nicolas Vuillamy Date: Sun, 6 Feb 2022 14:27:33 +0100 Subject: [PATCH] Manage Gitlab certificate (#1249) * Manage gitlab certificate * Build * Manage GITLAB_CUSTOM_CERTIFICATE quick build push * Fix python issue quick build push * Error msg doc * [MegaLinter] Apply linters fixes * Update config json schema quick build push Co-authored-by: nvuillam --- CHANGELOG.md | 6 +++ Dockerfile | 1 + docs/reporters/GitlabCommentReporter.md | 14 ++++--- flavors/ci_light/Dockerfile | 1 + flavors/dart/Dockerfile | 1 + flavors/documentation/Dockerfile | 1 + flavors/dotnet/Dockerfile | 1 + flavors/go/Dockerfile | 1 + flavors/java/Dockerfile | 1 + flavors/javascript/Dockerfile | 1 + flavors/php/Dockerfile | 1 + flavors/python/Dockerfile | 1 + flavors/ruby/Dockerfile | 1 + flavors/rust/Dockerfile | 1 + flavors/salesforce/Dockerfile | 1 + flavors/scala/Dockerfile | 1 + flavors/swift/Dockerfile | 1 + flavors/terraform/Dockerfile | 1 + .../megalinter-configuration.jsonschema.json | 21 ++++++++++ megalinter/reporters/GitlabCommentReporter.py | 40 +++++++++++++++---- 20 files changed, 84 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65f6c164073..a1bcbaf5f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] (beta, main branch content) +- Fixes + - Manage to use local certificate with Gitlab comments reporter using GITLAB_SSL_CERTIFICATE_PATH ([#1239](https://github.com/megalinter/megalinter/issues/1239)) + Note: Can be used with `megalinter/megalinter@beta` in your GitHub Action mega-linter.yml file, or with `megalinter/megalinter:beta` docker image +- Fixes + - Gitlab Comments Reporter: allow to use certificates with variable GITLAB_CUSTOM_CERTIFICATE (or GITLAB_CERTIFICATE_PATH only if [PRE_COMMANDS](https://megalinter.github.io/configuration/#pre-commands) are used) ([#1239](https://github.com/megalinter/megalinter/issues/1239)) + - Doc - Update images with screen records gifs - Add publish artifact task in azure pipelines doc diff --git a/Dockerfile b/Dockerfile index 3f826d08fe0..7a83fa0e650 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,7 @@ ARG PSSA_VERSION='latest' # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/docs/reporters/GitlabCommentReporter.md b/docs/reporters/GitlabCommentReporter.md index 4dd3b65eec5..30e434f1fdd 100644 --- a/docs/reporters/GitlabCommentReporter.md +++ b/docs/reporters/GitlabCommentReporter.md @@ -16,9 +16,11 @@ Click on hyperlinks to access detailed logs (click on **Download** in **Artifact ![config-gitlab-access-token](https://user-images.githubusercontent.com/17500430/151674446-1bcb1420-d9aa-4ae1-aaae-dcf51afb36ab.gif) -| Variable | Description | Default value | -|--------------------------------|-------------------------------------------------------------------------------------------|--------------------------| -| GITLAB_COMMENT_REPORTER | Activates/deactivates reporter | true | -| GITLAB_API_URL | URL where the github API can be reached
May be overridden if using self-hosted Gitlab | `https://api.gitlab.com` | -| GITLAB_SERVER_URL | URL of the Gitlab instance
May be overridden if using self-hosted Gitlab | `https://gitlab.com` | -| GITLAB_ACCESS_TOKEN_MEGALINTER | Must contain a Gitlab private access token defined with api access | | +| Variable | Description | Default value | +|--------------------------------|--------------------------------------------------------------------------------------------------------|--------------------------| +| GITLAB_COMMENT_REPORTER | Activates/deactivates reporter | true | +| GITLAB_API_URL | URL where the github API can be reached
May be overridden if using self-hosted Gitlab | `https://api.gitlab.com` | +| GITLAB_SERVER_URL | URL of the Gitlab instance
May be overridden if using self-hosted Gitlab | `https://gitlab.com` | +| GITLAB_ACCESS_TOKEN_MEGALINTER | Must contain a Gitlab private access token defined with api access | | +| GITLAB_CUSTOM_CERTIFICATE | SSL certificate value to connect to Gitlab | | +| GITLAB_CERTIFICATE_PATH | Path to SSL certificate to connect to Gitlab (if SSL cert has been manually defined with PRE_COMMANDS) | | diff --git a/flavors/ci_light/Dockerfile b/flavors/ci_light/Dockerfile index 5756dda8508..9a47040a3ad 100644 --- a/flavors/ci_light/Dockerfile +++ b/flavors/ci_light/Dockerfile @@ -33,6 +33,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/dart/Dockerfile b/flavors/dart/Dockerfile index ddbe743d55c..cdee430e5fd 100644 --- a/flavors/dart/Dockerfile +++ b/flavors/dart/Dockerfile @@ -36,6 +36,7 @@ ARG GLIBC_VERSION='2.31-r0' # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/documentation/Dockerfile b/flavors/documentation/Dockerfile index d8e6aaf059b..f763e880c8b 100644 --- a/flavors/documentation/Dockerfile +++ b/flavors/documentation/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/dotnet/Dockerfile b/flavors/dotnet/Dockerfile index 5516eedbf04..07240552ea5 100644 --- a/flavors/dotnet/Dockerfile +++ b/flavors/dotnet/Dockerfile @@ -40,6 +40,7 @@ ARG PSSA_VERSION='latest' # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/go/Dockerfile b/flavors/go/Dockerfile index 1e6669100f1..c705370d9db 100644 --- a/flavors/go/Dockerfile +++ b/flavors/go/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/java/Dockerfile b/flavors/java/Dockerfile index 148ec294d60..57995e0e8d6 100644 --- a/flavors/java/Dockerfile +++ b/flavors/java/Dockerfile @@ -36,6 +36,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/javascript/Dockerfile b/flavors/javascript/Dockerfile index 5f202fb5b1e..d96fee71e7c 100644 --- a/flavors/javascript/Dockerfile +++ b/flavors/javascript/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/php/Dockerfile b/flavors/php/Dockerfile index 82c52c0db08..02d944d2695 100644 --- a/flavors/php/Dockerfile +++ b/flavors/php/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/python/Dockerfile b/flavors/python/Dockerfile index e423854cb4d..a49ea861c51 100644 --- a/flavors/python/Dockerfile +++ b/flavors/python/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/ruby/Dockerfile b/flavors/ruby/Dockerfile index 8dd6afb9ecd..1951fa202d5 100644 --- a/flavors/ruby/Dockerfile +++ b/flavors/ruby/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/rust/Dockerfile b/flavors/rust/Dockerfile index f661904e4ad..5858fd061bb 100644 --- a/flavors/rust/Dockerfile +++ b/flavors/rust/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/salesforce/Dockerfile b/flavors/salesforce/Dockerfile index b67ccac6a23..d1e31e67c9a 100644 --- a/flavors/salesforce/Dockerfile +++ b/flavors/salesforce/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/scala/Dockerfile b/flavors/scala/Dockerfile index 1f05f018a39..23405a55e8c 100644 --- a/flavors/scala/Dockerfile +++ b/flavors/scala/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/swift/Dockerfile b/flavors/swift/Dockerfile index 419a7c74609..1dce003e2de 100644 --- a/flavors/swift/Dockerfile +++ b/flavors/swift/Dockerfile @@ -35,6 +35,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/flavors/terraform/Dockerfile b/flavors/terraform/Dockerfile index ca0ccff014d..f195445634c 100644 --- a/flavors/terraform/Dockerfile +++ b/flavors/terraform/Dockerfile @@ -39,6 +39,7 @@ FROM python:3.9.7-alpine3.13 # APK Packages used by mega-linter core architecture RUN apk add --update --no-cache \ bash \ + ca-certificates \ coreutils \ curl \ file \ diff --git a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json index 00aa2485532..1a7e33c8a79 100644 --- a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json +++ b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json @@ -3314,6 +3314,20 @@ "title": "GitHub Workspace", "type": "string" }, + "GITLAB_ACCESS_TOKEN_MEGALINTER": { + "$id": "#/properties/GITLAB_ACCESS_TOKEN_MEGALINTER", + "default": true, + "description": "Gitlab access token with API access (can be user or project Access token)", + "title": "Gitlab Access Token", + "type": "string" + }, + "GITLAB_CERTIFICATE_PATH": { + "$id": "#/properties/GITLAB_CERTIFICATE_PATH", + "default": true, + "description": "Path to SSL certificate if necessary", + "title": "Gitlab SSL Certificate path", + "type": "string" + }, "GITLAB_COMMENT_REPORTER": { "$id": "#/properties/GITLAB_COMMENT_REPORTER", "default": true, @@ -3321,6 +3335,13 @@ "title": "Activate Gitlab MR Comments reporter", "type": "boolean" }, + "GITLAB_CUSTOM_CERTIFICATE": { + "$id": "#/properties/GITLAB_CUSTOM_CERTIFICATE", + "default": true, + "description": "Store your SSL certificate string value in a CI env variable named GITLAB_CUSTOM_CERTIFICATE if you want it to be copied by MegaLinter in a local certificate file that will be used by python gitlab lib", + "title": "Gitlab SSL Certificate value", + "type": "string" + }, "GIT_FILTER_REGEX_EXCLUDE": { "$id": "#/properties/GIT_FILTER_REGEX_EXCLUDE", "title": "Excluding regex filter for GIT descriptor", diff --git a/megalinter/reporters/GitlabCommentReporter.py b/megalinter/reporters/GitlabCommentReporter.py index 63e0732db91..aaa1595ad10 100644 --- a/megalinter/reporters/GitlabCommentReporter.py +++ b/megalinter/reporters/GitlabCommentReporter.py @@ -7,6 +7,7 @@ import gitlab from megalinter import Reporter, config +from megalinter.pre_post_factory import run_command from megalinter.utils_reporter import build_markdown_summary @@ -47,16 +48,39 @@ def produce_report(self): action_run_url = config.get("CI_JOB_URL", "") p_r_msg = build_markdown_summary(self, action_run_url) - # Post comment on merge request if found + # Build gitlab options + gitlab_options = {} + # auth token if config.get("GITLAB_ACCESS_TOKEN_MEGALINTER", "") != "": - gl = gitlab.Gitlab( - gitlab_server_url, - private_token=config.get("GITLAB_ACCESS_TOKEN_MEGALINTER"), + gitlab_options["private_token"] = config.get( + "GITLAB_ACCESS_TOKEN_MEGALINTER" ) else: - gl = gitlab.Gitlab( - gitlab_server_url, job_token=config.get("CI_JOB_TOKEN") + gitlab_options["job_token"] = config.get("CI_JOB_TOKEN") + # Certificate management + gitlab_certificate_path = config.get("GITLAB_CERTIFICATE_PATH", "") + if config.get("GITLAB_CUSTOM_CERTIFICATE", "") != "": + # Certificate value defined in an ENV variable + cert_value = config.get("GITLAB_CUSTOM_CERTIFICATE") + gitlab_certificate_path = "/etc/ssl/certs/gitlab-cert.crt" + with open(gitlab_certificate_path, "w", encoding="utf-8") as cert_file: + cert_file.write(cert_value) + logging.debug( + f"Updated {gitlab_certificate_path} with certificate value {cert_value}" + ) + if gitlab_certificate_path != "": + # Update certificates and set cert path in gitlab options + run_command( + {"cwd": "root", "command": "update-ca-certificates"}, + "GitlabCommentReporter", + self.master, ) + gitlab_options["ssl_verify"] = gitlab_certificate_path + # Create gitlab connection + logging.debug( + f"[GitlabCommentReporter] Logging to {gitlab_server_url} with {str(gitlab_options)}" + ) + gl = gitlab.Gitlab(gitlab_server_url, **gitlab_options) # Get gitlab project try: project = gl.projects.get(gitlab_project_id) @@ -143,5 +167,7 @@ def display_auth_error(self, e): logging.error( "[Gitlab Comment Reporter] You may need to define a masked Gitlab CI/CD variable " "MEGALINTER_ACCESS_TOKEN containing a personal token with scope 'api'\n" - "(if already defined, your token is probably invalid)" + str(e) + "(if already defined, your token is probably invalid)\n" + "If you are using local certificate, you also may need to define variables " + "GITLAB_CUSTOM_CERTIFICATE or GITLAB_CERTIFICATE_PATH" + str(e) )