-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[WIT-682] MWAA SP does not validate provisioning request
# New features and improvements - Test coverage for MRs is now exported - Test coverage artifacts are now saved in the pipeline - CI optimizations # Bug fixes - Implemented proper validation for v1/validate endpoint - open-generator-cli uses a fixed version - Enabled akka.loglevel to INFO # Related issue Closes WIT-682 # Definition of Done for Feature/Hotfixes ## All Developments - [x] Feature was implemented as per the requirements - [x] If some code parts are complex they must be commented with code documentation - [x] CI/CD is successful - [x] Code coverage is not reduced, any new code is covered - [x] E2E/integration tests are successful (whether run locally or in CI/CD) - [x] If dependencies were changed, be sure that they will not impact the project, that their license is compatible, and that they introduce no vulnerabilities - [x] Documentation have been updated * Documentation has been updated with explanation of the new feature if it is user-facing (eg component now has additional setting) or it impacts him in some other way (eg optional field that becomes mandatory) * If it is a breaking change, we have documented it as such in the MR description in a "Breaking Changes" section - [x] Check that you are not affecting any existing environments with these changes, especially the Sandbox/Playground. This means that merging it to master and deploying it to these environments will not break them and **no manual operations that are not reported in the documentation will be needed** - [x] Check that nothing is out of order and nothing problematic is included in the changes (sensitive information, credentials, customer information or other intellectual property) as they could end up being public (we have Open Source SP already published and automatically mirrored) - [x] Security, Authentication and Authorization have been considered. No SQL injection, tokens handling, RBAC integration. Common security vulnerabilities identified and resolved ## API Development - [x] Semantic of API has been checked, it is comprehensible, meaningful, with no redundant information and user oriented - [x] Meaningful unit and integration tests are present - [x] API Parameters are checked and errors are handled - [x] Returned errors are meaningful to the user - [x] API contract has been defined and documented. Documentation means explaining the meaning of all fields and including at least one example - [x] Exceptions and errors are handled, without letting the underlying framework to respond with a generic Internal Server Error - [x] API Performance has been assessed and is good for real world use cases. Database accesses have been optimized. - [x] API is logging in compliance with audit standards, presence of sensitive information for GDPR has been assessed and removed or managed in case is needed ## DB Development - [x] The database schema is designed to accurately represent the data model and meet the requirements - [x] Tables, relationships, and constraints (e.g. primary keys, foreign keys, unique constraints) are defined appropriately and following a common convention - [x] Normalization principles are applied to eliminate data redundancy and ensure data integrity - [x] Schema semantic is meaningful - [x] Fields naming are following naming convention ( Ex. camelCase or snake_case) - [x] No fields with mixed behaviors and meaning. If a field is representing an enum, enum values are strongly mutually exclusive - [x] Data Types have been reviewed and they are a good fit for each field - [x] Indexes are defined to optimize query performance for frequently accessed data, paying attention to do not affect too much write path and the overall complexity - [x] Sensitive data is stored securely, encrypted if required, and access is restricted to authorized users - [x] Backup and restore procedures have been updated to introduce new or changed tables - [x] Migration scripts to upgrade and downgrade the database have been implemented and tested
- Loading branch information
Cristian Astorino
authored and
Nicolò Bidotti
committed
Nov 9, 2023
1 parent
4271ae8
commit 5622332
Showing
30 changed files
with
821 additions
and
218 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,19 +6,7 @@ include: | |
| ref: 'main' | ||
| file: 'common/witboost.downstream.gitlab-ci.yml' | ||
|
|
||
| image: ubuntu:20.04 | ||
|
|
||
| before_script: | ||
| - apt-get update -yqq | ||
| - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata | ||
| - apt-get install -yqq openjdk-17-jdk-headless | ||
| - apt-get install -yqq gpg | ||
| - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list | ||
| - mkdir -p /root/.gnupg | ||
| - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823 | ||
| - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg | ||
| - apt-get update -yqq | ||
| - apt-get install -yqq sbt | ||
| image: sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.9_9_1.9.7_2.13.12 | ||
|
|
||
| variables: | ||
| SBT_OPTS: "-Dsbt.global.base=sbt-cache/sbtboot -Dsbt.boot.directory=sbt-cache/boot -Dsbt.ivy.home=sbt-cache/ivy -Dsbt.ci=true" | ||
|
|
@@ -33,7 +21,7 @@ cache: | |
|
|
||
| stages: | ||
| - setup | ||
| - checkFormatting | ||
| - check | ||
| - test | ||
| - build | ||
| - package | ||
|
|
@@ -52,12 +40,12 @@ setup: | |
| dotenv: vars.env | ||
|
|
||
| checkFormatting: | ||
| stage: checkFormatting | ||
| stage: check | ||
| script: | ||
| - 'sbt scalafmtSbtCheck scalafmtCheckAll' | ||
|
|
||
| witboost.helm.checks: | ||
| stage: checkFormatting | ||
| stage: check | ||
| extends: .witboost.helm.base-job | ||
| before_script: [] | ||
| cache: [] | ||
|
|
@@ -70,30 +58,35 @@ witboost.helm.checks: | |
| test: | ||
| stage: test | ||
| script: | ||
| - apt-get install -yqq npm | ||
| - npm install @openapitools/openapi-generator-cli -g | ||
| - 'sbt clean generateCode coverage test multi-jvm:test coverageReport' | ||
| - apt-get update -yqq && apt-get install -yqq npm | ||
| - npm install @openapitools/[email protected] -g | ||
| - 'sbt clean generateCode coverage test coverageReport' | ||
| coverage: '/Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)/' | ||
| artifacts: | ||
| paths: | ||
| - target/scala-2.13/scoverage-report/* | ||
| - target/scala-2.13/coverage-report/* | ||
| reports: | ||
| coverage_report: | ||
| coverage_format: cobertura | ||
| path: 'target/scala-2.13/coverage-report/cobertura.xml' | ||
|
|
||
| build: | ||
| services: | ||
| - docker:19.03.12-dind | ||
| - docker:24.0.5-dind | ||
| stage: build | ||
| variables: | ||
| DOCKER_HOST: tcp://docker:2375 | ||
| script: | | ||
| apt-get install -yqq curl | ||
| curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg | ||
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu focal stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null | ||
| apt-get update -yqq | ||
| apt-get install -yqq docker-ce-cli | ||
| apt-get install -yqq npm | ||
| npm install @openapitools/openapi-generator-cli -g | ||
| apt-get update -yqq && apt-get install -yqq ca-certificates curl gnupg npm | ||
| install -m 0755 -d /etc/apt/keyrings | ||
| curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && chmod a+r /etc/apt/keyrings/docker.gpg | ||
| echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null | ||
| apt-get update -yqq && apt-get install -yqq docker-ce-cli | ||
| npm install @openapitools/[email protected] -g | ||
| echo $VERSION | ||
| docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY | ||
| sbt clean generateCode compile k8tyGitlabCIPublish docker:publish | ||
| artifacts: | ||
| reports: | ||
| dotenv: vars.env | ||
| witboost.helm.deploy: | ||
| stage: package | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
8 changes: 8 additions & 0 deletions
8
src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/common/StringUtils.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| package it.agilelab.datamesh.mwaaspecificprovisioner.common | ||
|
|
||
| object StringUtils { | ||
|
|
||
| implicit class StringImplicits(val s: String) { | ||
| def ensureTrailingSlash: String = if (s.endsWith("/")) s else s"$s/" | ||
| } | ||
| } |
5 changes: 5 additions & 0 deletions
5
src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/error/ErrorType.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package it.agilelab.datamesh.mwaaspecificprovisioner.error | ||
|
|
||
| trait ErrorType { | ||
| def errorMessage: String | ||
| } |
9 changes: 9 additions & 0 deletions
9
src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/error/ProvisionErrorType.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| package it.agilelab.datamesh.mwaaspecificprovisioner.error | ||
|
|
||
| import it.agilelab.datamesh.mwaaspecificprovisioner.s3.gateway.S3GatewayError | ||
|
|
||
| case class ProvisionErrorType(error: S3GatewayError) extends ErrorType { | ||
|
|
||
| override def errorMessage: String = | ||
| s"An error occurred while provisioning/unprovisioning the component. Details: ${error.getMessage}" | ||
| } |
47 changes: 47 additions & 0 deletions
47
src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/error/ValidationErrorType.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package it.agilelab.datamesh.mwaaspecificprovisioner.error | ||
|
|
||
| import cats.data.NonEmptyList | ||
|
|
||
| trait ValidationErrorType extends ErrorType | ||
|
|
||
| case class InvalidDescriptor(errors: NonEmptyList[String]) extends ValidationErrorType { | ||
| override def errorMessage: String = s"Descriptor is not valid. Details: ${errors.toList.mkString(",")}" | ||
| } | ||
|
|
||
| case class InvalidComponent(componentId: String) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The component '$componentId' to provision is not present" | ||
| } | ||
|
|
||
| case class InvalidComponentId(componentId: String) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The componentId '$componentId' is not valid" | ||
| } | ||
|
|
||
| case class InvalidDagName(fieldName: String, error: Throwable) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The $fieldName field is not present or is invalid. Details: ${error.getMessage}" | ||
| } | ||
|
|
||
| case class InvalidDestinationPath(fieldName: String, error: Throwable) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The $fieldName field is not present or is invalid. Details: ${error.getMessage}" | ||
| } | ||
|
|
||
| case class InvalidSourcePath(fieldName: String, error: Throwable) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The $fieldName field is not present or is invalid. Details: ${error.getMessage}" | ||
| } | ||
|
|
||
| case class InvalidBucketName(fieldName: String, error: Throwable) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The $fieldName field is not present or is invalid. Details: ${error.getMessage}" | ||
| } | ||
|
|
||
| case class InvalidScheduleCron(fieldName: String, error: Throwable) extends ValidationErrorType { | ||
| override def errorMessage: String = s"The $fieldName field is not present or is invalid. Details: ${error.getMessage}" | ||
| } | ||
|
|
||
| case class ErrorSourceFile(bucket: String, key: String, error: Throwable) extends ValidationErrorType { | ||
|
|
||
| override def errorMessage: String = | ||
| s"An error occurred while verifying existence of the file $key in bucket $bucket: ${error.getMessage}" | ||
| } | ||
|
|
||
| case class MissingSourceFile(bucket: String, key: String) extends ValidationErrorType { | ||
| override def errorMessage: String = s"File $key not found in bucket $bucket" | ||
| } |
Oops, something went wrong.