From 767af7de07ed8f39ef37d5ada3ab4eefaf4b8ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sch=C3=BCtte?= Date: Sat, 8 Jun 2024 11:26:33 +0200 Subject: [PATCH 1/4] support multi-doc yaml files, used for GitLab 17+ spec header --- test-input/.gitlab-ci-with-spec.yml | 12 +++++++++ yaml_shellcheck.py | 38 ++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 test-input/.gitlab-ci-with-spec.yml diff --git a/test-input/.gitlab-ci-with-spec.yml b/test-input/.gitlab-ci-with-spec.yml new file mode 100644 index 0000000..7a2c928 --- /dev/null +++ b/test-input/.gitlab-ci-with-spec.yml @@ -0,0 +1,12 @@ +spec: + inputs: + target: + type: string + description: The Target URL +--- +trigger: + stage: .post + image: $CI_REGISTRY/pipeline-components/helperimages:alpine-utils + script: + - | + curl --location $[[ inputs.target ]] diff --git a/yaml_shellcheck.py b/yaml_shellcheck.py index 0f468bd..6188acf 100755 --- a/yaml_shellcheck.py +++ b/yaml_shellcheck.py @@ -308,30 +308,46 @@ def get_shell_tasks(data, path): return result -def select_yaml_schema(data, filename): +def select_yaml_schema(documents, filename): # try to determine CI system and file format, - # returns the right get function + # returns the right get function, and the document index to read + + if len(documents) < 1: + raise ValueError(f"read {filename}, no valid YAML document, this should never happen") + + # special case first: GitLab 17 adds an optional spec-document before the main content document + # https://docs.gitlab.com/ee/ci/yaml/inputs.html + if len(documents) == 2 and "spec" in documents[0]: + logging.info(f"read {filename} as GitLab CI config with spec header section ...") + return get_gitlab_scripts, 1 + + # in previous versions we ignored additional documents in YAML files + if len(documents) > 1: + logging.warning(f"{filename} contains multiple YAML, only the first will be checked") + + # else: documents == 1; all other tools and cases only read a single YAML document + data = documents[0] if isinstance(data, dict) and "pipelines" in data: logging.info(f"read {filename} as Bitbucket Pipelines config...") - return get_bitbucket_scripts + return get_bitbucket_scripts, 0 elif isinstance(data, dict) and "on" in data and "jobs" in data: logging.info(f"read {filename} as GitHub Actions config...") - return get_github_scripts + return get_github_scripts, 0 elif isinstance(data, dict) and "version" in data and "jobs" in data: logging.info(f"read {filename} as CircleCI config...") - return get_circleci_scripts + return get_circleci_scripts, 0 elif ( isinstance(data, dict) and "steps" in data and "kind" in data and "type" in data ): logging.info(f"read {filename} as Drone CI config...") - return get_drone_scripts + return get_drone_scripts, 0 elif isinstance(data, list): logging.info(f"read {filename} as Ansible file...") - return get_ansible_scripts + return get_ansible_scripts, 0 elif isinstance(data, dict): # TODO: GitLab is the de facto default value, we should add more checks here logging.info(f"read {filename} as GitLab CI config...") - return get_gitlab_scripts + return get_gitlab_scripts, 0 else: raise ValueError(f"read {filename}, cannot determine CI tool from YAML structure") @@ -365,9 +381,9 @@ def from_yaml(cls, constructor, node): yaml.register_class(GitLabReference) with open(filename, "r") as f: - data = yaml.load(f) - get_script_snippets = select_yaml_schema(data, filename) - return get_script_snippets(data) + yaml_documents = list(yaml.load_all(f)) + get_script_snippets, document_index = select_yaml_schema(yaml_documents, filename) + return get_script_snippets(yaml_documents[document_index]) def write_tmp_files(args, data): From ecb091840a895e187511b7643085cbf51b06261a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sch=C3=BCtte?= Date: Sat, 8 Jun 2024 11:55:29 +0200 Subject: [PATCH 2/4] replace GitLab input parameter with dummy variable --- yaml_shellcheck.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yaml_shellcheck.py b/yaml_shellcheck.py index 6188acf..d0132aa 100755 --- a/yaml_shellcheck.py +++ b/yaml_shellcheck.py @@ -251,6 +251,9 @@ def flatten_nested_string_lists(data): for section in ["script", "before_script", "after_script"]: if section in data[jobkey]: script = data[jobkey][section] + script = flatten_nested_string_lists(script) + # replace inputs interpolation with dummy variable + script = re.sub(r'\$\[\[\s*(inputs\.[^\s>]*)\s*]]', r'$INPUT_PARAMETER', script) result[f"{jobkey}/{section}"] = flatten_nested_string_lists(script) return result From 107cc75dbb83221ae454932b15b667057be2c960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sch=C3=BCtte?= Date: Sat, 8 Jun 2024 12:08:38 +0200 Subject: [PATCH 3/4] test: gitlab input parameter in job name --- test-input/.gitlab-ci-with-spec.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test-input/.gitlab-ci-with-spec.yml b/test-input/.gitlab-ci-with-spec.yml index 7a2c928..d00ecb6 100644 --- a/test-input/.gitlab-ci-with-spec.yml +++ b/test-input/.gitlab-ci-with-spec.yml @@ -3,8 +3,11 @@ spec: target: type: string description: The Target URL + dest_env: + type: string + description: The Target environment --- -trigger: +$[[ inputs.dest_env ]]-trigger: stage: .post image: $CI_REGISTRY/pipeline-components/helperimages:alpine-utils script: From 356fb2324e669f6cb22f0cdbe5315266c42d8c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sch=C3=BCtte?= Date: Sat, 8 Jun 2024 12:11:14 +0200 Subject: [PATCH 4/4] gitlab input parameter with functions --- test-input/.gitlab-ci-with-spec.yml | 2 +- yaml_shellcheck.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-input/.gitlab-ci-with-spec.yml b/test-input/.gitlab-ci-with-spec.yml index d00ecb6..4807b60 100644 --- a/test-input/.gitlab-ci-with-spec.yml +++ b/test-input/.gitlab-ci-with-spec.yml @@ -12,4 +12,4 @@ $[[ inputs.dest_env ]]-trigger: image: $CI_REGISTRY/pipeline-components/helperimages:alpine-utils script: - | - curl --location $[[ inputs.target ]] + curl --location $[[ inputs.target | expand_vars | truncate(5,8) ]] diff --git a/yaml_shellcheck.py b/yaml_shellcheck.py index d0132aa..5266258 100755 --- a/yaml_shellcheck.py +++ b/yaml_shellcheck.py @@ -253,7 +253,7 @@ def flatten_nested_string_lists(data): script = data[jobkey][section] script = flatten_nested_string_lists(script) # replace inputs interpolation with dummy variable - script = re.sub(r'\$\[\[\s*(inputs\.[^\s>]*)\s*]]', r'$INPUT_PARAMETER', script) + script = re.sub(r'\$\[\[\s*(inputs\.[^]]*)\s*]]', r'$INPUT_PARAMETER', script) result[f"{jobkey}/{section}"] = flatten_nested_string_lists(script) return result