From 039cdb30e9141f0b6198c0780f98e7bae4ac568d Mon Sep 17 00:00:00 2001 From: Saurabh Shrihar Date: Sat, 13 Apr 2024 12:00:04 +0400 Subject: [PATCH 1/2] Fix for working with public workflow repo --- .DS_Store | Bin 6148 -> 6148 bytes .github/CODEOWNERS | 2 - .github/ISSUE_TEMPLATE/bug_report.md | 41 ----- .github/ISSUE_TEMPLATE/feature_request.md | 26 --- .github/PULL_REQUEST_TEMPLATE.md | 16 +- .github/workflows/codeql.yml | 2 +- .../workflows/ecs_deploy_docker_taskdef.yaml | 77 +++++--- .gitignore | 3 +- README.md | 75 +------- Support/taskdef_template/Pipfile | 7 + Support/taskdef_template/Pipfile.lock | 76 ++++++++ .../taskdef_template/example_parameters.yaml | 22 +++ Support/taskdef_template/taskdef_creator.py | 174 ++++++++++++++++++ .../taskdef_template/taskdef_template.json | 149 +++++++++++++++ action.yml | 140 -------------- docs/gcp-build-pipeline.md | 87 --------- sonar-project.properties | 4 +- 17 files changed, 494 insertions(+), 407 deletions(-) delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 Support/taskdef_template/Pipfile create mode 100644 Support/taskdef_template/Pipfile.lock create mode 100644 Support/taskdef_template/example_parameters.yaml create mode 100644 Support/taskdef_template/taskdef_creator.py create mode 100644 Support/taskdef_template/taskdef_template.json delete mode 100644 action.yml delete mode 100644 docs/gcp-build-pipeline.md diff --git a/.DS_Store b/.DS_Store index 119f3cf7c15b06043997de43b83a01e872ff0fed..afed36b9b329a15ca1422dcf3d0817c69dcf5b48 100644 GIT binary patch delta 82 zcmZoMXfc=|#>B`mu~2NHo}wrV0|Nsi1A_nqgD-B)qu~2NHo}wr#0|Nsi1A_nqLq0> $GITHUB_ENV + echo "taskdef_file_template=pipelines/Support/taskdef_template/taskdef_template.json" >> $GITHUB_ENV + else + echo "taskdef_file_script=${{inputs.taskdef_file_script}}" >> $GITHUB_ENV + echo "taskdef_file_template=${{inputs.taskdef_file_template}}" >> $GITHUB_ENV + fi - name: Create taskdef file dynamically using parameters passed run: | - ls -lrt - export PIPENV_PIPFILE=.github/taskdefinition_template/Pipfile - python -m pip install --upgrade pip && \ - pip install pipenv && \ - pipenv install && \ - pipenv run python ${{ inputs.taskdef_file_script }} \ - ${{ inputs.taskdef_file_vars }} ${{ inputs.taskdef_file_template }} + script_dir="$(dirname ${{ env.taskdef_file_script }})" + if [[ -z "$script_dir" ]]; then + echo "Warning: Script name doesn't contain a directory path. Using current directory." + script_dir="." + fi + export PIPENV_PIPFILE=$script_dir/Pipfile + python -m pip install --upgrade pip + pip install pipenv + pipenv install + pipenv run python ${{ env.taskdef_file_script }} \ + -pf code/${{ inputs.taskdef_file_vars }} -tt ${{ env.taskdef_file_template }} \ + -acc ${{ inputs.account_number }} -app ${{ inputs.app_name }} - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ inputs.aws_region }} - role-to-assume: arn:aws:iam::${{ inputs.account_number }}:role/${{ inputs.app_name }}-GithubActionsRole + role-to-assume: | + arn:aws:iam::${{ inputs.account_number }}:role/${{ inputs.app_name }}-GithubActionsRole role-session-name: GithubActionsSession - name: Login to Amazon ECR @@ -73,6 +108,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} ECR_REPOSITORY: ${{ inputs.app_name }}-ecr run: | + cd code current_dir=$(realpath .) target_dir=$(dirname "$(realpath "${{ inputs.docker_file }}")") if [[ "$current_dir" != "$target_dir" ]]; then @@ -83,11 +119,8 @@ jobs: fi docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f $docker_file_basename . docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - - - name: Extract directory path - run: | - echo "TASKDEF_FILE=$(dirname "${{ inputs.taskdef_file_template }}")/${{ inputs.app_name }}.json" >> $GITHUB_ENV + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_ENV + echo "TASKDEF_FILE=$(dirname "${{ env.taskdef_file_template }}")/${{ inputs.app_name }}.json" >> $GITHUB_ENV - name: Fill in the new image ID in the Amazon ECS task definition id: task-def @@ -95,7 +128,7 @@ jobs: with: task-definition: "${{ env.TASKDEF_FILE }}" container-name: ${{ inputs.app_name }} - image: ${{ steps.build-image.outputs.image }} + image: ${{ env.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v1 diff --git a/.gitignore b/.gitignore index 75ec3f0..b334fd9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.vscode/* \ No newline at end of file +.vscode/* +*.pyc \ No newline at end of file diff --git a/README.md b/README.md index 0fb612a..f75850e 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,7 @@ - -![Build][Build-badge] -[![Coverage][Coverage-badge]][Sonar-url] -[![Vulnerabilities][Vulnerability-badge]][Sonar-url] +# Pipelines -# 0xPolygon Pipelines -This repo serves as the repository for shared pipelines across the Polygon organization. To learn more about using -shared pipelines, please see the [Shared Pipelines Documentation](https://docs.github.com/en/actions/creating-actions/sharing-actions-and-workflows-with-your-organization). +Supports github workflow pipelines by providing solutions for build pipelines by tracking common workflows. +The workflow are executed through passing required parameters. -### Built With - -![Static Badge](https://img.shields.io/badge/alcohol-sarcasm-8A2BE2?logo=polygon) - -## Getting Started - -### Local Development - -## Usage - -To use this workflow, provide the required inputs when triggering the workflow run. Ensure that the necessary secrets and permissions are configured in your GitHub repository for GCP authentication and Docker image pushing. - - steps: - - id: gcp-build-action - uses: 0xPolygon/pipelines@v1 - with: - workload_identity_provider: ${{ env.WIF_PROVIDER }} - service_account: ${{ env.WIF_SERVICE_ACCOUNT }} - gar_location: ${{ env.GAR_LOCATION }} - docker_image: ${{ env.IMAGE_NAME }} - dockerfile_name: Dockerfile - dockerfile_path: . - critical_count: ${{ env.CRITICAL_COUNT }} - helm_values_path: './helm-chart/values.yaml' - attestor: ${{ env.ATTESTOR }} - attestor_project: ${{ env.ATTESTOR_PROJECT_ID }} - keyversion_project: ${{ env.ATTESTOR_PROJECT_ID }} - keyversion_location: ${{ env.GAR_LOCATION }} - keyversion_keyring: ${{ env.KEY_RING }} - keyversion_key: ${{ env.KEY }} - -Read more info: [gcp-build-pipeline](/docs/gcp-build-pipeline.md) - -## Contributing - -This is the place to document your delivery workflow. For example: - -1. Clone the project -2. Create a feature branch beginning with the ticket number (`git checkout -b INC-7689/update-readme`) -3. Commit your changes (`git commit -m 'Update README.me with default template`) -4. Push to the branch (`git push origin INC-7689/update-readme`) -5. Open a Pull Request -6. After review and approval, changes are deployed immediately - -## Contact - -![Email][Email-badge] -![Slack][Slack-badge] - - - -[Build-badge]: https://github.com/0xPolygon/learn-api/actions/workflows/main.yml/badge.svg -[Coverage-badge]: https://sonarqube.polygon.technology/api/project_badges/measure?project=TODO -[Vulnerability-badge]: https://sonarqube.polygon.technology/api/project_badges/measure?project=TODO -[Sonar-url]: https://sonarqube.polygon.technology/dashboard?id=TODO -[Language-badge]: https://img.shields.io/badge/Nodejs-18.0-informational -[Language-url]: https://nodejs.org/en -[Email-badge]: https://img.shields.io/badge/Email-devops@polygon.technology-informational?logo=gmail -[Slack-badge]: https://img.shields.io/badge/Slack-team_devops-informational?logo=slack -[Production-badge]: https://img.shields.io/badge/Production_URL-polygon.technology-informational -[Production-url]: https://link.to/prod -[Staging-badge]: https://img.shields.io/badge/Staging_URL-staging.polygon.technology-informational -[Staging-url]: https://link.to/staging +## Support +Support folder tracks scripts that can be used to support the execution workflow for github pipelines. \ No newline at end of file diff --git a/Support/taskdef_template/Pipfile b/Support/taskdef_template/Pipfile new file mode 100644 index 0000000..c5754f7 --- /dev/null +++ b/Support/taskdef_template/Pipfile @@ -0,0 +1,7 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] +pyyaml = "6.0.1" \ No newline at end of file diff --git a/Support/taskdef_template/Pipfile.lock b/Support/taskdef_template/Pipfile.lock new file mode 100644 index 0000000..8f83fa2 --- /dev/null +++ b/Support/taskdef_template/Pipfile.lock @@ -0,0 +1,76 @@ +{ + "_meta": { + "hash": { + "sha256": "50b136775148391a355082540f8cf183843fd6305f19e0c822e1741ed4d6a6c8" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "pyyaml": { + "hashes": [ + "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", + "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", + "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==6.0.1" + } + }, + "develop": {} +} diff --git a/Support/taskdef_template/example_parameters.yaml b/Support/taskdef_template/example_parameters.yaml new file mode 100644 index 0000000..6efa0f3 --- /dev/null +++ b/Support/taskdef_template/example_parameters.yaml @@ -0,0 +1,22 @@ +--- +region: eu-west-1 +account_number: "123456789" +hostport: 3000 +containerport: 3000 +app_name: appname +role: backend +environment: staging +iac: aws-test-applications-eu-west-1-apps-appname +team_name: dev-experience +memory: 1024 +cpu: 512 +env_vars: + - name: START_BLOCK + value: "0" + - name: NODE_ENV + value: "staging" +secret_vars: + - KAFKA_CONNECTION_URL + - MONGO_URL + - RPC_WS_ENDPOINT_URL_LIST + - SENTRY_DSN diff --git a/Support/taskdef_template/taskdef_creator.py b/Support/taskdef_template/taskdef_creator.py new file mode 100644 index 0000000..13b41a7 --- /dev/null +++ b/Support/taskdef_template/taskdef_creator.py @@ -0,0 +1,174 @@ +"""Generates taskefinition file for the github workflow to deploy +""" + +import argparse +import json +import os +import re +import yaml + + +class TaskdefCreator: + """Handles creation of taskdef file for ECS using template""" + + def __init__(self): + parser = argparse.ArgumentParser(description="Task definition creator") + parser.add_argument("-pf", "--parameters_file", type=str, + help="Parameters yaml file with required values", + ) + parser.add_argument("-tt", "--taskdef_template", type=str, + help="Template json file to be used" + ) + parser.add_argument("-acc", "--account_number", type=str, + help="AWS account number to be used for deployment", + default="" + ) + parser.add_argument("-app", "--app_name", type=str, + help="ECS app name", default="" + ) + self.args = parser.parse_args() + self.template_data = "" + self.taskdef_final_file = "" + + def _read_yaml_file(self): + """Reads yaml file into dictionary from user input""" + with open(self.args.parameters_file, "r") as file_object: + try: + data = yaml.safe_load(file_object) + return data + except yaml.YAMLError as error: + print(f"Error reading YAML file {self.args.parameters_file}: {error}") + return None + + def _read_template_file(self): + """Reads template file for data substitution""" + try: + with open(self.args.taskdef_template, "r") as file: + self.template_data = file.read() + except FileNotFoundError: + print(f"Error: File '{self.args.taskdef_template}' not found.") + except IOError as error: + print(f"Error reading file '{self.args.taskdef_template}': {error}") + except Exception as error: + print(f"An unexpected error occurred: {error}") + + def _substitute_env_vars(self, data_read: list): + """Substitutes value in self.template_data based on env names and values + + Args: + data_read (list): [{name: value}...] of environment variables for app + """ + env_values = "" + env_template = """{ + "name": "name_sub", + "value": "value_sub" + }, + """ + for env_data in data_read: + name = env_data.get("name") + value = env_data.get("value") + env_values += env_template.replace("name_sub", name).replace( + "value_sub", value + ) + env_values = env_values.strip().rstrip(",") + self.template_data = self.template_data.replace("$env_vars", env_values) + + def _substitute_secret_vars(self, data_read: list): + """Substitutes value in self.template_data based on secret names + + Args: + data_read (list): [{name: value}...] of environment variables for app + """ + secret_str = "" + secret_template = """{ + "valueFrom": "arn:aws:ssm:$region:$account_number:parameter/$app_name/$secret_name", + "name": "$secret_name" + },""" + for secret in data_read: + secret_str += secret_template.replace("$secret_name", secret) + secret_str = secret_str.strip().rstrip(",") + self.template_data = self.template_data.replace("$secret_vars", secret_str) + + def _subtitute_data(self, user_data: dict, sub: str): + """Subtitutes data in self.template_data based on user specified data + + Args: + user_data (dict): Data read from user defined yaml file + sub (str): Element to be searched for and substituted + """ + data_read = str(user_data.get(sub, "")).strip() + if sub == "app_name": + if not data_read and self.args.app_name: + data_read = self.args.app_name + self.template_data = self.template_data.replace(f"${sub}", data_read) + if sub == "app_name": + self.taskdef_final_file = data_read + + def _print_secrets_to_create(self, json_data_str: str): + """Prints secrets to be created for systems manager parameter store + + Args: + json_data_str (str): File data for taskdef file + """ + for line in json_data_str.split("\n"): + if "valueFrom" in line: + secret = ( + line.split('"valueFrom": "arn:aws:ssm:', 1)[1].strip().rstrip('",') + ) + + print(f"Update SSM for secret: {secret}") + + def create_taskdef_file(self, account_number: str): + """Create a taskdef file based on the app name""" + directory = os.path.dirname(self.args.taskdef_template) + file_path = os.sep.join([directory, self.taskdef_final_file]) + ".json" + self.template_data = self.template_data.replace("\n", "") + self.template_data = re.sub(r"\s+", " ", self.template_data) + json_data_dict = json.loads(self.template_data) + json_data_str = json.dumps(json_data_dict, indent=2) + self._print_secrets_to_create(json_data_str) + with open(file_path, "w") as file_object: + json.dump(json_data_dict, file_object, ensure_ascii=False, indent=2) + print(f"Created file: {file_path}") + with open(file_path, "r") as fp: + print("File data:", fp.read().replace(account_number, "****")) + + def substitute_values(self): + """Substitutes values taskdef template to generate a new file + Expected strings in template to be replaced for values are: + region, account_number, hostport, containerport, app_name, + role, environment, iac, team_name, memory, cpu, env_vars, secret_vars + """ + self._read_template_file() + user_data = self._read_yaml_file() + expected_sub = [ + "region", + "account_number", + "hostport", + "containerport", + "app_name", + "role", + "environment", + "iac", + "team_name", + "memory", + "cpu", + ] + if "account_number" not in user_data: + user_data["account_number"] = self.args.account_number + + if user_data: + if user_data.get("account_number") is None and self.args.account_number: + user_data["account_number"] = self.args.account_number + self._substitute_env_vars(user_data.get("env_vars", [])) + self._substitute_secret_vars(user_data.get("secret_vars", [])) + [user_data.pop(key) for key in ["env_vars", "secret_vars"] if key in user_data] + for sub in expected_sub: + self._subtitute_data(user_data, sub) + return user_data["account_number"] + + +if __name__ == "__main__": + TASKDEF_CREATOR = TaskdefCreator() + ACCOUNT_NUMBER = TASKDEF_CREATOR.substitute_values() + TASKDEF_CREATOR.create_taskdef_file(ACCOUNT_NUMBER) diff --git a/Support/taskdef_template/taskdef_template.json b/Support/taskdef_template/taskdef_template.json new file mode 100644 index 0000000..7f2e97e --- /dev/null +++ b/Support/taskdef_template/taskdef_template.json @@ -0,0 +1,149 @@ +{ + "requiresCompatibilities": [ + "FARGATE" + ], + "inferenceAccelerators": [], + "containerDefinitions": [ + { + "dnsSearchDomains": null, + "environmentFiles": [], + "logConfiguration": { + "logDriver": "awsfirelens", + "options": { + "Name": "datadog", + "Host": "http-intake.logs.datadoghq.com", + "dd_service": "$app_name", + "dd_source": "nodejs", + "TLS": "on", + "provider": "ecs" + }, + "secretOptions": [ + { + "name": "apiKey", + "valueFrom": "arn:aws:ssm:$region:$account_number:parameter/DATADOG_APIKEY" + } + ] + }, + "entryPoint": null, + "portMappings": [ + { + "name": "$app_name", + "hostPort": $hostport, + "protocol": "tcp", + "containerPort": $containerport, + "appProtocol": "http" + } + ], + "command": null, + "linuxParameters": null, + "cpu": 0, + "environment": [ + $env_vars + ], + "resourceRequirements": null, + "ulimits": null, + "dnsServers": null, + "mountPoints": null, + "workingDirectory": null, + "secrets": [ + $secret_vars + ], + "dockerSecurityOptions": null, + "memory": null, + "memoryReservation": null, + "volumesFrom": null, + "stopTimeout": null, + "image": "nginx:latest", + "startTimeout": null, + "firelensConfiguration": null, + "dependsOn": null, + "disableNetworking": null, + "interactive": null, + "healthCheck": null, + "essential": true, + "links": null, + "hostname": null, + "extraHosts": null, + "pseudoTerminal": null, + "user": null, + "readonlyRootFilesystem": null, + "dockerLabels": null, + "systemControls": null, + "privileged": null, + "name": "$app_name", + "repositoryCredentials": { + "credentialsParameter": "" + } + }, + { + "essential": true, + "image": "amazon/aws-for-fluent-bit:stable", + "name": "log_router", + "firelensConfiguration": { + "type": "fluentbit", + "options": { + "enable-ecs-log-metadata": "true" + } + }, + "environment": null, + "secrets": null, + "memoryReservation": 50, + "resourceRequirements": null, + "portMappings": [], + "environmentFiles": [], + "mountPoints": null, + "volumesFrom": null, + "hostname": null, + "user": null, + "workingDirectory": null, + "extraHosts": null, + "logConfiguration": null, + "ulimits": null, + "dockerLabels": null, + "dependsOn": null, + "repositoryCredentials": { + "credentialsParameter": "" + } + } + ], + "volumes": [], + "networkMode": "awsvpc", + "memory": "$memory", + "cpu": "$cpu", + "executionRoleArn": "arn:aws:iam::$account_number:role/$app_name-TaskRole", + "family": "$app_name-taskdefinition", + "taskRoleArn": "arn:aws:iam::$account_number:role/$app_name-TaskRole", + "runtimePlatform": { + "operatingSystemFamily": "LINUX" + }, + "tags": [ + { + "key": "Role", + "value": "$role" + }, + { + "key": "Environment", + "value": "$environment" + }, + { + "key": "Service", + "value": "$app_name.polygon.technology" + }, + { + "key": "Host", + "value": "AWS" + }, + { + "key": "IAC", + "value": "$iac" + }, + { + "key": "Team", + "value": "$team_name" + }, + { + "key": "Name", + "value": "$app_name-taskdefinition" + } + ] +} \ No newline at end of file diff --git a/action.yml b/action.yml deleted file mode 100644 index 1c7be9e..0000000 --- a/action.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: 'GCP Build Pipeline' -description: 'Build and push Docker image to Google Cloud Artifact Registry' -inputs: - workload_identity_provider: - description: 'Full identifier of Workload Identity Provider (e.g., project number, pool name, provider name).' - required: true - service_account: - description: 'Email address or unique identifier of the Google Cloud service account for which to generate credentials.' - required: true - gar_location: - description: 'Google Cloud Artifact Registry location.' - required: true - docker_image: - description: 'Full name of the Docker image.' - required: true - dockerfile_name: - description: 'Name of the Dockerfile.' - required: true - default: 'Dockerfile' - dockerfile_path: - description: 'Path to the Dockerfile.' - required: true - default: '.' - critical_count: - description: 'Critical vulnerabilities count.' - required: true - default: '5' - helm_values_path: - description: 'Path to the Helm values file for configuration.' - required: true - default: './helm-charts/values.yaml' - attestor: - description: 'Name of the attestor to be used for signing the docker images.' - required: true - attestor_project: - description: 'Google Cloud Platform (GCP) project where the attestor is located.' - required: true - keyversion_project: - description: 'GCP project where the key version is stored.' - required: true - keyversion_location: - description: 'Location/region of the key version.' - required: true - keyversion_keyring: - description: 'Keyring associated with the key version.' - required: true - keyversion_key: - description: 'Key associated with the key version.' - required: true - -runs: - using: "composite" - steps: - - uses: actions/checkout@v4 - - uses: google-github-actions/setup-gcloud@v1 - - - id: auth - uses: google-github-actions/auth@v1 - with: - token_format: "access_token" - workload_identity_provider: ${{ inputs.workload_identity_provider }} - service_account: ${{ inputs.service_account }} - - - id: docker-auth - uses: docker/login-action@v1 - with: - username: "oauth2accesstoken" - password: ${{ steps.auth.outputs.access_token }} - registry: ${{ inputs.gar_location }}-docker.pkg.dev - - - id: build-docker-image - run: |- - docker build -t "${{ inputs.docker_image }}:${{ github.sha }}" -f ${{ inputs.dockerfile_name }} ${{ inputs.dockerfile_path }} - shell: bash - - - id: push-docker-image - run: |- - docker push "${{ inputs.docker_image }}:${{ github.sha }}" - shell: bash - - - id: scan-vulnerabilities - run: |- - (gcloud artifacts docker images scan "${{ inputs.docker_image }}:${{ github.sha }}" --format="value(response.scan)" --remote --quiet) > ./scan_id.txt - shell: bash - - - id: check-critical-vulnerabilities - run: |- - #!/bin/bash - # Check if the scan_id.txt file exists - if [ ! -f ./scan_id.txt ]; then - echo "Error: scan_id.txt not found." - exit 1 - fi - # Use gcloud to list vulnerabilities and check for CRITICAL severity - severity=$(gcloud artifacts docker images list-vulnerabilities \ - "$(cat ./scan_id.txt)" \ - --format="value(vulnerability.effectiveSeverity)") - - # Check if CRITICAL vulnerability is found - chk=$(echo "$severity" | grep -c "CRITICAL") - if [ "$chk" -gt ${{ inputs.critical_count }} ]; then - echo "Failed vulnerability check for CRITICAL level" - exit 1 - else - echo "No CRITICAL vulnerability found. Congratulations!" - exit 0 - fi - shell: bash - - - id: sign-docker-image - run: |- - export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - gcloud components install beta --quiet - - DIGEST=$(gcloud container images describe ${{ inputs.docker_image }}:${{ github.sha }} --format='get(image_summary.digest)') - - gcloud beta container binauthz attestations sign-and-create \ - --artifact-url="${{ inputs.docker_image }}@${DIGEST}" \ - --attestor="${{ inputs.attestor }}" \ - --attestor-project="${{ inputs.attestor_project }}" \ - --keyversion-project="${{ inputs.keyversion_project }}" \ - --keyversion-location="${{ inputs.keyversion_location }}" \ - --keyversion-keyring="${{ inputs.keyversion_keyring }}" \ - --keyversion-key="${{ inputs.keyversion_key }}" \ - --keyversion="1" - shell: bash - - - id: update-helm-values - run: |- - DIGEST=$(gcloud container images describe ${{ inputs.docker_image }}:${{ github.sha }} \ - --format='get(image_summary.digest)') - - sed -i "s|image:.*|image: ${{ inputs.docker_image }}@${DIGEST}|" ${{ inputs.helm_values_path }} - shell: bash - - - id: push-back - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Apply automatic changes to Update image repository in Helm values - file_pattern: ${{ inputs.helm_values_path }} diff --git a/docs/gcp-build-pipeline.md b/docs/gcp-build-pipeline.md deleted file mode 100644 index 7002332..0000000 --- a/docs/gcp-build-pipeline.md +++ /dev/null @@ -1,87 +0,0 @@ -# GCP Build Pipeline - -## Overview -This GitHub Actions workflow sets up a build pipeline for Docker images on Google Cloud Platform (GCP) with the following key features: - -- Building and pushing Docker images to Google Cloud Artifact Registry. -- Scanning Docker images for vulnerabilities and checking for critical severity. -- Signing Docker images for Binary Authorization. -- Updating Helm chart values with the latest Docker image details. -- Automatically committing changes to the Helm values file. - -The custom action code is in `action.yml` file in the root directory of this repo. - -## Inputs -The workflow accepts the following parameters: - -- `workload_identity_provider`: Full identifier of the Workload Identity Provider. -- `service_account`: Email address or unique identifier of the Google Cloud service account. -- `gar_location`: Google Cloud Artifact Registry location. -- `docker_image`: Full name of the Docker image. -- `dockerfile_name`: Name of the Dockerfile (default: 'Dockerfile'). -- `dockerfile_path`: Path to the Dockerfile (default: '.'). -- `critical_count`: Critical vulnerabilities count (default: '5'). -- `helm_values_path`: Path to the Helm values file for configuration (default: './helm-charts/values.yaml'). -- `attestor`: Name of the attestor for signing Docker images. -- `attestor_project`: GCP project where the attestor is located. -- `keyversion_project`: GCP project where the key version is stored. -- `keyversion_location`: Location/region of the key version. -- `keyversion_keyring`: Keyring associated with the key version. -- `keyversion_key`: Key associated with the key version. - -## Workflow Steps -1. **Checkout Code:** Uses `actions/checkout` to fetch the source code. -2. **Set up GCP CLI:** Uses `google-github-actions/setup-gcloud` to configure the Google Cloud CLI. -3. **Authenticate:** Authenticates with GCP using the specified service account and workload identity provider. -4. **Docker Login:** Logs in to the Google Cloud Artifact Registry using the provided credentials. -5. **Build Docker Image:** Builds the Docker image with the specified Dockerfile. -6. **Push Docker Image:** Pushes the Docker image to the Google Cloud Artifact Registry. -7. **Scan Vulnerabilities:** Scans the pushed Docker image for vulnerabilities. -8. **Check Critical Vulnerabilities:** Checks if the number of critical vulnerabilities exceeds the specified count. -9. **Sign Docker Image:** Signs the Docker image using Binary Authorization. -10. **Update Helm Values:** Updates the Helm chart values with the latest Docker image details. -11. **Push Back Changes:** Automatically commits changes to the Helm values file. - -## Notes -- The workflow utilizes Google Cloud CLI and Docker commands for building, pushing, and scanning Docker images. -- Binary Authorization is used to sign Docker images for security. -- Helm chart values are updated with the latest Docker image details automatically. - -## Usage -To use this workflow, provide the required inputs when triggering the workflow run. Ensure that the necessary secrets and permissions are configured in your GitHub repository for GCP authentication and Docker image pushing. - - steps: - - id: custom-action - uses: 0xPolygon/pipelines@v1 - with: - workload_identity_provider: ${{ env.WIF_PROVIDER }} - service_account: ${{ env.WIF_SERVICE_ACCOUNT }} - gar_location: ${{ env.GAR_LOCATION }} - docker_image: ${{ env.IMAGE_NAME }} - dockerfile_name: Dockerfile - dockerfile_path: . - critical_count: ${{ env.CRITICAL_COUNT }} - helm_values_path: './helm-chart/values.yaml' - attestor: ${{ env.ATTESTOR }} - attestor_project: ${{ env.ATTESTOR_PROJECT_ID }} - keyversion_project: ${{ env.ATTESTOR_PROJECT_ID }} - keyversion_location: ${{ env.GAR_LOCATION }} - keyversion_keyring: ${{ env.KEY_RING }} - keyversion_key: ${{ env.KEY }} - -## Updating the custom action - -In the future, if you like to update the pipeline code, you can follow the procedure outlined below: - -* Create a feature branch from the main branch. -* Modify the _action.yml_ file within the feature branch and commit the changes. -* Assign a tag to the committed branch, such as `v2-test-feature`. -* Push the changes to GitHub along with the assigned tag. -* In the other repository, employ the action using the following syntax: `uses: 0xPolygon/pipelines@v2-test-feature ` and verify that everything is functioning as expected. -* If all tests pass, merge the changes into the main branch and create a new release tag, for instance, `v2`. -* Subsequently, you can utilize this custom action by referencing uses: `0xPolygon/pipelines@v2`. -* You can delete the tags that used for testing (`v2-test-feature`). - -## References - -* Implementing custom action: https://docs.github.com/en/actions/creating-actions/creating-a-composite-action diff --git a/sonar-project.properties b/sonar-project.properties index 303d284..f88c24a 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,2 +1,2 @@ -sonar.projectKey=0xPolygon_pipelines #replace with repository name -sonar.organization=0xpolygon #replace with github org name +sonar.projectKey=0xPolygon_pipelines +sonar.organization=0xpolygon From 9ea7cedfe28f1c8f8ab7d29ed56a277d27de2573 Mon Sep 17 00:00:00 2001 From: Saurabh Shrihar Date: Fri, 26 Apr 2024 16:52:19 +0400 Subject: [PATCH 2/2] Remove line from security review concern --- Support/taskdef_template/taskdef_creator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Support/taskdef_template/taskdef_creator.py b/Support/taskdef_template/taskdef_creator.py index 13b41a7..2228ba5 100644 --- a/Support/taskdef_template/taskdef_creator.py +++ b/Support/taskdef_template/taskdef_creator.py @@ -116,8 +116,6 @@ def _print_secrets_to_create(self, json_data_str: str): line.split('"valueFrom": "arn:aws:ssm:', 1)[1].strip().rstrip('",') ) - print(f"Update SSM for secret: {secret}") - def create_taskdef_file(self, account_number: str): """Create a taskdef file based on the app name""" directory = os.path.dirname(self.args.taskdef_template)