diff --git a/CITATION.cff b/CITATION.cff index 413b491c6..1cdc899a9 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,34 +1,38 @@ cff-version: 1.2.0 -message: "If you use this software, please cite it as below." -authors: -- family-names: "Talasila" - given-names: "Prasad" - orcid: "https://orcid.org/0000-0002-8973-2640" title: "Digital Twin as a Service" -version: 0.3.0 -date-released: 2023-11-08 +version: 0.6.0 +date-released: 2024-11-11 url: "https://github.com/into-cps-association/DTaaS" +message: Please cite the following works when using this software. preferred-citation: - conference: "The 2023 IEEE International Conference on Digital Twin" authors: - - family-names: "Talasila" - given-names: "Prasad" - orcid: "https://orcid.org/0000-0002-8973-2640" - - family-names: "Gomes" - given-names: "Cláudio" - orcid: "https://orcid.org/0000-0000-0000-0000" - - family-names: "Mikkelsen" - given-names: "Peter Høgh" - orcid: "https://orcid.org/0000-0003-2321-758X" - - family-names: "Gil" - given-names: "Santiago" - orcid: "https://orcid.org/0000-0002-1789-531X" - - family-names: "Kamburjan" - given-names: "Eduard" - orcid: "https://orcid.org/0000-0002-0996-2543" - - family-names: "Larsen" - given-names: "Peter Gorm" - orcid: "https://orcid.org/0000-0002-4589-1500" - title: "Digital Twin as a Service (DTaaS): A Platform for Digital Twin Developers and Users" - month: 8 - year: 2023 \ No newline at end of file + - family-names: Talasila + given-names: Prasad + - family-names: Gomes + given-names: Cláudio + - family-names: Vosteen + given-names: Lars B + - family-names: Iven + given-names: Hannes + - family-names: Leucker + given-names: Martin + - family-names: Gil + given-names: Santiago + - family-names: Mikkelsen + given-names: Peter H + - family-names: Kamburjan + given-names: Eduard + - family-names: Larsen + given-names: Peter G + doi: 10.1177/00375497241298653 + identifiers: + - type: doi + value: 10.1177/00375497241298653 + title: Composable digital twins on Digital Twin as a Service platform + date-published: 2024-01-01 + year: 2024 + journal: SIMULATION + publisher: + name: 'SAGE Publications Sage UK: London, England' + start: '00375497241298653' + type: article \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..3276197f4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +Open new issue. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/README.md b/README.md index 802456a28..d3830d0fb 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,15 @@ if there is a suggestion to improve the software. If you find this repo useful for your research, please consider citing our paper: ```bibtex -@INPROCEEDINGS{talasila2023dtaas, - author={Talasila, Prasad and Gomes, Cláudio and Mikkelsen, Peter Høgh and Arboleda, Santiago Gil and Kamburjan, Eduard and Larsen, Peter Gorm}, - booktitle={2023 IEEE Smart World Congress (SWC)}, - title={Digital Twin as a Service (DTaaS): A Platform for Digital Twin Developers and Users}, - year={2023}, - pages={1-8}, - keywords={digital twins;physical twin;automation;life cycle;composition}, - doi={10.1109/SWC57546.2023.10448890}} +@article{talasila2024composable, +author = {Prasad Talasila and Cl{\'a}udio Gomes and Lars B Vosteen and Hannes Iven and Martin Leucker and Santiago Gil and Peter H Mikkelsen and Eduard Kamburjan and Peter G Larsen}, +title ={Composable digital twins on Digital Twin as a Service platform}, +journal = {SIMULATION}, +pages = {00375497241298653}, +year={2024}, +doi = {10.1177/00375497241298653}, +publisher = {SAGE Publications Sage UK: London, England} +} ``` ## :hammer_and_wrench: Development Setup diff --git a/SECURITY.md b/SECURITY.md index a0919533c..44f9e8fbd 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,7 +6,8 @@ We are currently providing security updates for: | Version | Supported | | ------- | ------------------ | -| 0.5.0 | :white_check_mark: | +| 0.6.0 | :white_check_mark: | +| 0.5.0 | :x: | | 0.4.0 | :x: | | 0.3.0 | :x: | | 0.2.0 | :x: | diff --git a/deploy/services/gitlab/.env b/deploy/services/gitlab/.env index a15c5e774..84ad69fd2 100644 --- a/deploy/services/gitlab/.env +++ b/deploy/services/gitlab/.env @@ -1,2 +1,2 @@ -GITLAB_HOME='/Users//DTaaS/deploy/services/gitlab' +DTAAS_DIR='/Users//DTaaS' SERVER_DNS='foo.com' diff --git a/deploy/services/gitlab/INTEGRATION.md b/deploy/services/gitlab/INTEGRATION.md index 2ce0b73c3..e900d0e2d 100644 --- a/deploy/services/gitlab/INTEGRATION.md +++ b/deploy/services/gitlab/INTEGRATION.md @@ -66,7 +66,7 @@ the following files: _deploy/config/client/env.local.js_. 1. _deploy/docker/.env.local_ Add localpath and username. -If the DTaaS application is hosted at , then configure +If the DTaaS application is hosted at , then configure the following files: 1. **DTaaS Client Authorization** token in diff --git a/deploy/services/gitlab/README.md b/deploy/services/gitlab/README.md index c8ede730f..0ec211f03 100644 --- a/deploy/services/gitlab/README.md +++ b/deploy/services/gitlab/README.md @@ -29,7 +29,7 @@ Edit the `.env` file available in this directory to contain the following variab | Variable | Example Value | Explanation | | :---------- | :------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------- | -| GITLAB_HOME | '/home/Desktop/DTaaS/deploy/services/gitlab' | Full path to the DTaaS gitlab directory. This is an absolute path with no trailing slash. | +| DTAAS_DIR | '/Users//DTaaS' | Full path to the DTaaS directory. This is an absolute path with no trailing slash. | | SERVER_DNS | either `foo.com` or `localhost` | The server DNS, if you are deploying with a dedicated server. Remember not use _http(s)_ at the beginning of the DNS string. | **NOTE**: The DTaaS client uses the `react-oidc-context` node package, which @@ -114,5 +114,7 @@ for further help. ## Pending Tasks This README helps with installation of Gitlab along side DTaaS application. -But the OAuth2 integration between Gitlab and DTaaS will still be still pending. -Follow the [INTEGRATION](./INTEGRATION.md) guide to setup the Gitlab integration. +But the OAuth2 integration between Gitlab and DTaaS will still be pending. +Follow the [integration guide](./INTEGRATION.md) and +[runner setup guide](../runner/GITLAB-RUNNER.md) to setup the Gitlab +integration. diff --git a/deploy/services/gitlab/compose.gitlab.yml b/deploy/services/gitlab/compose.gitlab.yml index bd1588a31..0c066afb3 100644 --- a/deploy/services/gitlab/compose.gitlab.yml +++ b/deploy/services/gitlab/compose.gitlab.yml @@ -15,9 +15,9 @@ services: nginx['redirect_http_to_https'] = false letsencrypt['enable'] = false volumes: - - '${GITLAB_HOME}/config:/etc/gitlab' - - '${GITLAB_HOME}/logs:/var/log/gitlab' - - '${GITLAB_HOME}/data:/var/opt/gitlab' + - '${DTAAS_DIR}/deploy/services/gitlab/config:/etc/gitlab' + - '${DTAAS_DIR}/deploy/services/gitlab/logs:/var/log/gitlab' + - '${DTAAS_DIR}/deploy/services/gitlab/data:/var/opt/gitlab' shm_size: '256m' labels: - "traefik.enable=true" diff --git a/deploy/services/runner/GITLAB-RUNNER.md b/deploy/services/runner/GITLAB-RUNNER.md new file mode 100644 index 000000000..ab4d75427 --- /dev/null +++ b/deploy/services/runner/GITLAB-RUNNER.md @@ -0,0 +1,150 @@ +# GitLab Runner Integration + +This document outlines the steps needed to create a Docker container named +`gitlab-runner` which will contain a single runner that will be responsible for +the execution of Digital Twins. There are two installation scenarios: + +1. __Localhost Installation__ - You are using the integrated runner locally with + a GitLab instance hosted at `https://localhost/gitlab`. +1. __Server Installation__ - You are using the integrated runner with a GitLab + instance hosted on a production server. This server may be a remote server + and not necessarily your own, and may have TLS enabled with a self-signed + certificate. + +Following the steps below sets up the integrated runner which can be used to +execute digital twins from the Digital Twins Preview Page. + +## Prerequisites + +A GitLab Runner picks up CI/CD jobs by communicating with a GitLab instance. +For an explanation of how to set up a GitLab instance that integrates with a +DTaaS application, refer to [our GitLab instance document](../gitlab/README.md) +and [our GitLab integration guide](../gitlab/INTEGRATION.md). + +The rest of this document assumes you have a running DTaaS application with a +GitLab instance running. + +## Runner Scopes + +A GitLab Runner can be configured for three different scopes: + +| Runner Scope | Description | +|-----------------|-------------| +| Instance Runner | Available to all groups and projects in a GitLab instance. | +| Group Runner | Available to all projects and subgroups in a group. | +| Project Runner | Associated with one specific project. | + +We suggest creating __instance runners__ as they are the most straightforward, but +any type will work. More about these three types can be found on +[the official GitLab documentation page](https://docs.gitlab.com/ee/ci/runners/runners_scope.html). + +## Obtaining A Registration Token + +First, we will obtain the token necessary to register the runner for the GitLab +instance. Open your GitLab instance (remote or local) and depending on your +choice of runner scope, follow the steps given below: + +| Runner Scope | Steps | +|-----------------|-------| +| Instance Runner |1. On the __Admin__ dashboard, navigate to __CI/CD > Runners__.
2. Select __New instance runner__.| +| Group Runner |1. On the __DTaaS__ group page, navigate to __Settings > CI/CD > Runners__.
2. Ensure the __Enable shared runners for this group__ option is enabled.
3. On the __DTaaS__ group page, navigate to __Build > Runners__.
4. Select __New group runner__.| +| Project Runner |1. On the __DTaaS__ group page, select the project named after your GitLab username.
2. Navigate to __Settings > CI/CD > Runners__.
3. Select __New project runner__.| + +For any scope you have chosen, you will be directed to a page to create a +runner: + +1. Under __Platform__, select the Linux operating system. +1. Under __Tags__, add a `linux` tag. +1. Select __Create runner__. + +You should then see the following screen: + +![Runner Registration Screen](./runner-registration.png) + +Be sure to save the generated runner authentication token. + +## Configuring the Runner + +Depending on your installation scenario, the runner setup reads certain +configurations settings: + +1. __Localhost Installation__ - uses `deploy/docker/.env.local` +1. __Server Installation__ - uses `deploy/docker/.env.server` + +These files are integral to running the DTaaS application, so it will be +assumed that you have already configured these. + +We need to register the runner with the GitLab instance so that they may +communicate with each other. `deploy/services/runner/runner-config.toml` +has the following template: + +```toml +[[runners]] + name = "dtaas-runner-1" + url = "https://foo.com/gitlab/" # Edit this + token = "xxx" # Edit this + executor = "docker" + [runners.docker] + tls_verify = false + image = "ruby:2.7" + privileged = false + disable_entrypoint_overwrite = false + oom_kill_disable = false + volumes = ["/cache"] + network_mode = "host" # Disable this in secure contexts +``` + +1. Set the `url` variable to the URL of your GitLab instance. +1. Set the `token` variable to the runner registration token you obtained earlier. +1. If you are following the server installation scenario, remove the line + `network_mode = "host"`. + +A list of advanced configuration options is provided on the +[GitLab documentation page](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). + +## Start the GitLab Runner + +You may use the following commands to start and stop the `gitlab-runner` +container respectively, depending on your installation scenario: + +1. Localhost Installation + + ```bash + docker compose -f deploy/services/runner/compose.runner.local.yml --env-file deploy/docker/.env.local up -d + docker compose -f deploy/services/runner/compose.runner.local.yml --env-file deploy/docker/.env.local down + ``` + +1. Server Installation + + ```bash + docker compose -f deploy/services/runner/compose.runner.server.yml --env-file deploy/docker/.env.server up -d + docker compose -f deploy/services/runner/compose.runner.server.yml --env-file deploy/docker/.env.server down + ``` + +Once the container starts, the runner within it will run automatically. You can +tell if the runner is up and running by navigating to the page where +you created the runner. For example, an Instance Runner would look like this: + +![Status indicator under Admin Area > Runners](./runner-activation.png) + +You will now have a GitLab runner ready to accept jobs for the GitLab instance. + +## Advanced: Runner Executor + +The term `runner` could refer one of two things: + +1. The `gitlab-runner` Container + This is the Docker container that is created when you execute the commands + given above. It is based on the `gitlab/gitlab-runner:alpine` image, and is + used to spawn one or more _executors_ that actually execute the CI/CD jobs. + + These executors are spawned using a packaged version of Docker within the + `gitlab-runner` image. +1. The Runner Executor + These are agents spawned by the `gitlab-runner` container, not as children + but as __siblings__ of the container. These runner executors will not show up + on running commands such as `docker ps`, but you can check their status by + running `gitlab-runner status` inside the `gitlab-runner` container. + +Keeping this distinction in mind is important, as the GitLab documentation +sometimes uses them interchangeably. diff --git a/deploy/services/runner/compose.runner.local.yml b/deploy/services/runner/compose.runner.local.yml new file mode 100644 index 000000000..74ff35b49 --- /dev/null +++ b/deploy/services/runner/compose.runner.local.yml @@ -0,0 +1,15 @@ +# Environment variables taken from deploy/docker/.env.local + +services: + gitlab-runner: + container_name: gitlab-runner + # Runner image version is independent of the gitlab-ce image version + image: gitlab/gitlab-runner:alpine-v17.5.3 + volumes: + - "./runner-config.toml:/etc/gitlab-runner/config.toml:ro" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + # Maps the self-signed certificate for localhost to the container + - "${DTAAS_DIR}/deploy/docker/certs/localhost/fullchain.pem:/etc/gitlab-runner/certs/localhost.crt:ro" + + # To make https://localhost accessible from the container + network_mode: host diff --git a/deploy/services/runner/compose.runner.server.yml b/deploy/services/runner/compose.runner.server.yml new file mode 100644 index 000000000..b3fec82fc --- /dev/null +++ b/deploy/services/runner/compose.runner.server.yml @@ -0,0 +1,13 @@ +# Environment variables taken from deploy/docker/.env.server + +services: + gitlab-runner: + container_name: gitlab-runner + # Runner image version is independent of the gitlab-ce image version + image: gitlab/gitlab-runner:alpine-v17.5.3 + volumes: + - "./runner-config.toml:/etc/gitlab-runner/config.toml:ro" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + # Maps the self-signed certificate for your server to the container + # Remove this if you are not using a self-signed certificate + - "${DTAAS_DIR}/deploy/docker/certs/${SERVER_DNS}/fullchain.pem:/etc/gitlab-runner/certs/${SERVER_DNS}.crt:ro" diff --git a/deploy/services/runner/runner-activation.png b/deploy/services/runner/runner-activation.png new file mode 100644 index 000000000..93dc265db Binary files /dev/null and b/deploy/services/runner/runner-activation.png differ diff --git a/deploy/services/runner/runner-config.toml b/deploy/services/runner/runner-config.toml new file mode 100644 index 000000000..ab3bcbcad --- /dev/null +++ b/deploy/services/runner/runner-config.toml @@ -0,0 +1,13 @@ +[[runners]] + name = "dtaas-runner-1" + url = "https://foo.com/gitlab/" # Edit this + token = "xxx" # Edit this + executor = "docker" + [runners.docker] + tls_verify = false + image = "ruby:2.7" + privileged = false + disable_entrypoint_overwrite = false + oom_kill_disable = false + volumes = ["/cache"] + network_mode = "host" # Disable this in secure contexts diff --git a/deploy/services/runner/runner-registration.png b/deploy/services/runner/runner-registration.png new file mode 100644 index 000000000..85a82a624 Binary files /dev/null and b/deploy/services/runner/runner-registration.png differ diff --git a/docs/admin/gitlab/index.md b/docs/admin/gitlab/index.md index daf3dc3b7..6b94f769c 100644 --- a/docs/admin/gitlab/index.md +++ b/docs/admin/gitlab/index.md @@ -138,6 +138,6 @@ for further help. ## Pending Tasks This README helps with installation of Gitlab along side DTaaS application. -But the OAuth2 integration between Gitlab and DTaaS will still be still pending. -Follow the [integration guide](integration.md) and -[runner setup](runner.md) to setup the Gitlab integration. +But the OAuth2 integration between Gitlab and DTaaS will still be pending. +Follow the [integration guide](integration.md) and the +[runner setup guide](runner.md) to setup the Gitlab integration. diff --git a/docs/admin/gitlab/pipeline-token.PNG b/docs/admin/gitlab/pipeline-token.PNG new file mode 100644 index 000000000..03a699eb8 Binary files /dev/null and b/docs/admin/gitlab/pipeline-token.PNG differ diff --git a/docs/admin/gitlab/runner-activation.png b/docs/admin/gitlab/runner-activation.png new file mode 100644 index 000000000..93dc265db Binary files /dev/null and b/docs/admin/gitlab/runner-activation.png differ diff --git a/docs/admin/gitlab/runner-registration.png b/docs/admin/gitlab/runner-registration.png new file mode 100644 index 000000000..85a82a624 Binary files /dev/null and b/docs/admin/gitlab/runner-registration.png differ diff --git a/docs/admin/gitlab/runner.md b/docs/admin/gitlab/runner.md index 42d4c71c8..5076f5da7 100644 --- a/docs/admin/gitlab/runner.md +++ b/docs/admin/gitlab/runner.md @@ -1,74 +1,143 @@ -# Gitlab Runner Integration +# GitLab Runner Integration -To properly use the __Digital Twins Preview Page__, you need to -configure at least one project runner in your GitLab profile. -The first step is to configure the CI/CD pipeline in gitlab project. -The second step is to install the runner and integrate it -with the selected gitlab project. +This document outlines the steps needed to create a Docker container named +`gitlab-runner` which will contain a single runner that will be responsible for +the execution of Digital Twins. There are two installation scenarios: -## Configure Gitlab Project +1. __Localhost Installation__ - You are using the integrated runner locally with + a GitLab instance hosted at `https://localhost/gitlab`. +1. __Server Installation__ - You are using the integrated runner with a GitLab + instance hosted on a production server. This server may be a remote server + and not necessarily your own, and may have TLS enabled with a self-signed + certificate. -Follow the steps below: +Following the steps below sets up the integrated runner which can be used to +execute digital twins from the Digital Twins Preview Page. -1. Navigate to the _DTaaS_ group and select the project named after your - GitLab username. -1. In the project menu, go to Settings and select CI/CD. -1. Expand the __Runners__ section and click on _New project runner_. Follow the - configuration instructions carefully: - - Add __linux__ as a tag during configuration. - - Click on _Create runner_. A runner authentication token is generated. - This token will be used later for registering a runner. +## Prerequisites -## Runner +A GitLab Runner picks up CI/CD jobs by communicating with a GitLab instance. +For an explanation of how to set up a GitLab instance that integrates with a +DTaaS application, refer to [our GitLab instance document](./index.md) +and [our GitLab integration guide](./integration.md). -### Install Runner +The rest of this document assumes you have a running DTaaS application with a +GitLab instance running. -A detailed guide on installation of -[gitlab runners](https://docs.gitlab.com/runner/install/) -on Linux OS is available on -[gitlab website](https://docs.gitlab.com/runner/install/linux-repository.html). -Remember to use `linux` as tag for the runner. +## Runner Scopes -### Register Runner +A GitLab Runner can be configured for three different scopes: -Please see this [gitlab guide](https://docs.gitlab.com/runner/register/) -on registering a runner. +| Runner Scope | Description | +|-----------------|-------------| +| Instance Runner | Available to all groups and projects in a GitLab instance. | +| Group Runner | Available to all projects and subgroups in a group. | +| Project Runner | Associated with one specific project. | -Remember to choose _docker_ as executor and _ruby:2.7_ as -the default docker image. +We suggest creating __instance runners__ as they are the most straightforward, but +any type will work. More about these three types can be found on +[the official GitLab documentation page](https://docs.gitlab.com/ee/ci/runners/runners_scope.html). -```bash -$sudo gitlab-runner register --url https://gitlab.foo.com \ - --token xxxxx -``` +## Obtaining A Registration Token -Or, you can also register the runner in non-interactive mode by running +First, we will obtain the token necessary to register the runner for the GitLab +instance. Open your GitLab instance (remote or local) and depending on your +choice of runner scope, follow the steps given below: -```bash -$sudo gitlab-runner register \ - --non-interactive \ - --url "https://gitlab.foo.com/" \ - --token "xxxx" \ - --executor "docker" \ - --docker-image ruby:2.7 \ - --description "docker-runner" -``` +| Runner Scope | Steps | +|-----------------|-------| +| Instance Runner |1. On the __Admin__ dashboard, navigate to __CI/CD > Runners__.
2. Select __New instance runner__.| +| Group Runner |1. On the __DTaaS__ group page, navigate to __Settings > CI/CD > Runners__.
2. Ensure the __Enable shared runners for this group__ option is enabled.
3. On the __DTaaS__ group page, navigate to __Build > Runners__.
4. Select __New group runner__.| +| Project Runner |1. On the __DTaaS__ group page, select the project named after your GitLab username.
2. Navigate to __Settings > CI/CD > Runners__.
3. Select __New project runner__.| + +For any scope you have chosen, you will be directed to a page to create a +runner: + +1. Under __Platform__, select the Linux operating system. +1. Under __Tags__, add a `linux` tag. +1. Select __Create runner__. + +You should then see the following screen: + +![Runner Registration Screen](./runner-registration.png) -### Start Runner +Be sure to save the generated runner authentication token. -You can manually verify that the runner is available to pick up jobs by running -the following command: +## Configuring the Runner -```bash -$sudo gitlab-runner run +Depending on your installation scenario, the runner setup reads certain +configurations settings: + +1. __Localhost Installation__ - uses `deploy/docker/.env.local` +1. __Server Installation__ - uses `deploy/docker/.env.server` + +These files are integral to running the DTaaS application, so it will be +assumed that you have already configured these. + +We need to register the runner with the GitLab instance so that they may +communicate with each other. `deploy/services/runner/runner-config.toml` +has the following template: + +```toml +[[runners]] + name = "dtaas-runner-1" + url = "https://foo.com/gitlab/" # Edit this + token = "xxx" # Edit this + executor = "docker" + [runners.docker] + tls_verify = false + image = "ruby:2.7" + privileged = false + disable_entrypoint_overwrite = false + oom_kill_disable = false + volumes = ["/cache"] + network_mode = "host" # Disable this in secure contexts ``` -It can also be used to reactivate offline runners during subsequent sessions. +1. Set the `url` variable to the URL of your GitLab instance. +1. Set the `token` variable to the runner registration token you obtained earlier. +1. If you are following the server installation scenario, remove the line + `network_mode = "host"`. + +A list of advanced configuration options is provided on the +[GitLab documentation page](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). + +## Start the GitLab Runner + +You may use the following commands to start and stop the `gitlab-runner` +container respectively, depending on your installation scenario: + +1. Localhost Installation + + ```bash + docker compose -f deploy/services/runner/compose.runner.local.yml --env-file deploy/docker/.env.local up -d + docker compose -f deploy/services/runner/compose.runner.local.yml --env-file deploy/docker/.env.local down + ``` + +1. Server Installation + + ```bash + docker compose -f deploy/services/runner/compose.runner.server.yml --env-file deploy/docker/.env.server up -d + docker compose -f deploy/services/runner/compose.runner.server.yml --env-file deploy/docker/.env.server down + ``` + +Once the container starts, the runner within it will run automatically. You can +tell if the runner is up and running by navigating to the page where +you created the runner. For example, an Instance Runner would look like this: + +![Status indicator under Admin Area > Runners](./runner-activation.png) + +You will now have a GitLab runner ready to accept jobs for the GitLab instance. ## Pipeline Trigger Token -You also need to create a -[pipeline trigger token](https://archives.docs.gitlab.com/16.4/ee/ci/triggers/index.html). -This token is required to trigger pipelines by using the API. -You can create this token in your GitLab project's CI/CD settings under -the *Pipeline trigger tokens* section. \ No newline at end of file +The Digital Twins Preview Page uses the GitLab API which requires a +[Pipeline Trigger Token](https://docs.gitlab.com/ee/api/pipeline_triggers.html). +Go to your project in the __DTaaS__ group and navigate to +__Settings > CI/CD > Pipeline trigger tokens__. Add a new token with any +description of your choice. + +![Creating a Pipeline Trigger Token](./pipeline-token.PNG) + +You can now use the Digital Twins Preview Page to manage and execute your +digital twins. diff --git a/docs/developer/conduct.md b/docs/developer/conduct.md new file mode 100644 index 000000000..3276197f4 --- /dev/null +++ b/docs/developer/conduct.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +Open new issue. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/docs/developer/devops/api.md b/docs/developer/devops/api.md new file mode 100644 index 000000000..6e46dbe06 --- /dev/null +++ b/docs/developer/devops/api.md @@ -0,0 +1,30 @@ +# API Calls + +A GitLab DevOps pipeline can be triggered via an API call using a +[pipeline trigger token](https://docs.gitlab.com/ee/ci/triggers/) +which is created on the GitLab instance, with the following values: + +1. ``: The user GitLab trigger token. +1. ``: The name of the DT (e.g. mass-spring-damper). +1. ``: The specific tag of the GitLab runner that the user wants to use. +1. ``: The ID of the GitLab project, displayed in the project overview + page. + +The example given below sets the `DTName` variable to the desired DT name, +the `RunnerTag` variable to the specified GitLab Runner tag, and ensures the call +will be executed in the `main` branch: + +```bash + curl --request POST \ + --form "token=" \ + --form ref=main \ + --form "variables[DTName]=" \ + --form "variables[RunnerTag]=" \ + "https://maestro.cps.digit.au.dk/gitlab/api/v4/projects//trigger/pipeline" +``` + +--- + +Ref: Vanessa Scherma, Design and implementation of an integrated DevOps +framework for Digital Twins as a Service software platform, +Master's Degree Thesis, Politecnico Di Torino, 2024. diff --git a/docs/developer/devops/gitlab-ci-cd.md b/docs/developer/devops/gitlab-ci-cd.md new file mode 100644 index 000000000..3903acd34 --- /dev/null +++ b/docs/developer/devops/gitlab-ci-cd.md @@ -0,0 +1,215 @@ +# GitLab CI/CD Infrastructure + +Given that files in the Library are stored in a Git repository, the approach +employed was that of GitLab's parent-child pipelines. In this context, a parent +pipeline initiates the execution of another pipeline within the same project, +the latter of which is known as the child pipeline. More about pipelines can be +found in GitLab's documentation on +[CI/CD Pipelines](https://docs.gitlab.com/ee/ci/pipelines/). + +## CI/CD Pipelines + +Continuous Integration (CI) and Continuous Deployment (CD) represent two key +components of the DevOps methodology. + +CI involves frequent integration of code changes into a common repository. +Each integration triggers automated builds and tests that permit the detection +of issues at an early stage. This practice ensures that the changes made to +the code are checked fast enough, reducing the possibilities of integration +problems and hence ensuring high-quality software. + +CD automates the process of release, ensuring that code changes are +automatically tested and prepared for deployment. Teams using CD can deploy +updates rapidly and reliably, improving the responsiveness and quality of +software. Performed together, CI/CD automates the whole delivery pipeline for +software, increasing efficiency and reducing errors. They entirely eliminate, +or significantly reduce, the manual human input required for a code change to +be moved from a commit to a production environment. The entire process of +compilation, testing (including unit, integration, and regression testing), +deployment, and infrastructure provisioning is included. + +CI/CD practices are explained in more detail in +[this article by GitLab](https://about.gitlab.com/topics/ci-cd/). + +A CI/CD pipeline is a series of automated processes that manage CI and CD of +software. They are configured to run automatically, with no need for manual +intervention once activated. + +GitLab is a single application for the entire DevOps lifecycle, which means +it performs all of the basics required for CI/CD in one environment. The +[documentation provided by GitLab](https://docs.gitlab.com/ee/ci/pipelines/) +was instrumental in enabling a comprehensive understanding of the CI/CD +pipelines. + +Pipelines are composed of a number of essential components. _Jobs_ delineate +the specific tasks to be accomplished, while _stages_ define the sequence in +which jobs are executed. In this way, stages ensure that each step takes place +in the right order and make the pipeline more efficient and consistent. In the +event that all jobs within a stage are successfully completed, the pipeline +will automatically proceed to the subsequent stage. However, if any of the +jobs fail, the flow is interrupted without proceeding. + +When a pipeline is initiated, the jobs that have been defined within it are +then distributed among the available runners. + +GitLab runners are agents within the GitLab Runner application that execute +the jobs in accordance with their configuration and the available resources. +They can be configured to operate on a variety of platforms, including virtual +machines, containers, and physical servers. They can also be managed locally +or in a cloud environment. + +We use this GitLab parent-child pipeline setup to trigger execution of digital +twins stored in a user's GitLab repository. + +!!! note + The recommended practice is to modified these pipelines via the + [Digital Twins Preview Page](../../user/digital-twins/devops/ui.md). + +## Parent Pipeline + +The parent pipeline was configured as a top-level element. There is a single +stage called `triggers`, which is responsible for triggering other child +pipelines. + +In the `.gitlab-ci.yml` file, triggers are managed for DTs inside the user +repository. Each trigger is connected with one distinct DT and becomes active +when the corresponding value of the `DTName` variable is provided by the API +call. The `RunnerTag` variable is used to specify a custom runner tag that will +execute each job in the DT's pipeline. + +Below is an explanation of the keywords used in the CI/CD pipeline configuration: + +- **Image**: Specifies the Docker image, such as `fedora:41`, providing the + environment for the pipeline execution. +- **Stages**: Defines phases in the pipeline, such as triggers, organizing tasks + sequentially. +- **Trigger**: Initiates another pipeline or job, incorporating configurations + from an external file. +- **Include**: Imports configurations from another file for modular pipeline + setups. +- **Rules**: Sets conditions for job execution based on variables or states. +- **If**: A condition within rules specifying when a job should run based on the + value of a variable. +- **When**: Specifies the timing of job execution, such as `always`. +- **Variables**: Defines dynamic variables, like `RunnerTag`, used in the + pipeline. + +Here is an example of such a YAML file that registers a trigger for a DT named +`mass-spring-damper`: + +```yaml +image: fedora:41 + +stages: + - triggers + +trigger_mass-spring-damper: + stage: triggers + trigger: + include: digital_twins/mass-spring-damper/.gitlab-ci.yml + rules: + - if: '$DTName == "mass-spring-damper"' + when: always + variables: + RunnerTag: $RunnerTag +``` + +## Digital Twin Structure + +The `digital_twins` folder contains DTs that have been pre-built by one or +more users. The intention is that they should be sufficiently flexible to be +reconfigured as required for specific use cases. + +Let us look at an example of such a configuration. The +[dtaas/user1 repository on gitlab.com](https://gitlab.com/dtaas/user1) contains +the `digital_twins` directory with a `hello_world` example. Its file structure +looks like this: + +```text +hello_world/ +├── lifecycle/ +│ ├── clean +│ ├── create +│ ├── execute +│ └── terminate +├── .gitlab-ci.yml +└── description.md +``` + +The `lifecycle` directory here contains four files - `clean`, `create`, +`execute` and `terminate`, which are simple +[BASH scripts](https://www.gnu.org/software/bash/). These correspond to stages +in a digital twin's lifecycle. + +![Digital Twin Lifecycle](./images/dt-lifecycle.png) + +## Child Pipelines + +To automate the lifecycle of DT, a child pipeline has been incorporated into +its corresponding folder. Regardless of the image provided in the parent +pipeline, each child pipeline will use its own specified image specified in +its YAML configuration or Ruby's default image. + +The following are the explanations of the keywords used within the CI/CD child +pipeline based on +[GitLab's CI/CD YAML syntax reference](https://docs.gitlab.com/ee/ci/yaml/): + +1. `Stage` + It defines the steps that happen in a pipeline sequentially, for example, + create, execute and clean, to make sure that tasks occur in a specific order. +1. `Script` + It lists commands to be run at each step; for example, changing directories, + modifying permissions, or running lifecycle scripts. +1. `Tags` + It specifies which runner should run the jobs, thereby providing an + additional control over where and how the jobs are run. + +With the DT `mass-spring-damper` serving as a point of reference, +the stages in question are designed to facilitate the creation, execution, and +termination of the DT simulation, as well as the cleaning and restoration of the +environment to ensure its readiness for future executions. Here is an example of +a configuration that defines `create`, `execute` and `clean` as part of the +child pipeline: + +```yaml +image: ubuntu:20.04 + +stages: + - create + - execute + - clean + +create_mass-spring-damper: + stage: create + script: + - cd digital_twins/mass-spring-damper + - chmod +x lifecycle/create + - lifecycle/create + tags: + - $RunnerTag + +execute_mass-spring-damper: + stage: execute + script: + - cd digital_twins/mass-spring-damper + - chmod +x lifecycle/execute + - lifecycle/execute + tags: + - $RunnerTag + +clean_mass-spring-damper: + stage: clean + script: + - cd digital_twins/mass-spring-damper + - chmod +x lifecycle/terminate + - chmod +x lifecycle/clean + - lifecycle/terminate + tags: + - $RunnerTag +``` + +--- + +Ref: Vanessa Scherma, Design and implementation of an integrated DevOps +framework for Digital Twins as a Service software platform, +Master's Degree Thesis, Politecnico Di Torino, 2024. diff --git a/docs/developer/devops/images/devops-overview.png b/docs/developer/devops/images/devops-overview.png new file mode 100644 index 000000000..c93c0eb43 Binary files /dev/null and b/docs/developer/devops/images/devops-overview.png differ diff --git a/docs/developer/devops/images/dt-lifecycle.png b/docs/developer/devops/images/dt-lifecycle.png new file mode 100644 index 000000000..583fa2ddd Binary files /dev/null and b/docs/developer/devops/images/dt-lifecycle.png differ diff --git a/docs/developer/devops/implementation.md b/docs/developer/devops/implementation.md new file mode 100644 index 000000000..d4fd1afc1 --- /dev/null +++ b/docs/developer/devops/implementation.md @@ -0,0 +1,162 @@ +# Implemented Classes + +In order to facilitate the management of the lifecycle of DTs via the web +application interfaces, it was necessary to develop specific code within the +project client. The code was designed to facilitate efficient API calls through +the use of Gitbeaker as a wrapper, as this approach simplifies interactions +with GitLab’s REST API and reduces the complexity of the project code. + +The APIs have been integrated into the front-end by wiring up API endpoints to +front-end components, ensuring a seamless data flow. Unit and integration +testing was done to ensure the coverage of all functional requirements and solve +all problems regarding data consistency, performance, or user experience. + +Given below is our implementation of these classes in TypeScript: + +```typescript +class GitlabInstance { + async init(); + async getProjectId(); + async getTriggerToken(projectId: number); + async getDTSubfolders(projectId: number); + async getLibrarySubfolders( + projectId: number, + type: string, + isPrivate: boolean, + ); + executionLogs(); + async getPipelineJobs( + projectId: number, + pipelineId: number, + ); + async getJobTrace(projectId: number, jobId: number); + async getPipelineStatus( + projectId: number, + pipelineId: number, + ); +} + +class DigitalTwin { + async getDescription(); + async getFullDescription(); + private async triggerPipeline(); + async execute(); + async stop(projectId: number, pipeline: string); + async create( + files: FileState[], + cartAssets: LibraryAsset[], + libraryFiles: LibraryConfigFile[], + ); + async delete(); + async getDescriptionFiles(); + async getConfigFiles(); + async getLifecycleFiles(); + async prepareAllAssetFiles( + cartAssets: LibraryAsset[], + libraryFiles: LibraryConfigFile[], + ); + async getAssetFiles(); +} + +class LibraryAsset { + async getDescription(); + async getFullDescription(); + async getConfigFiles(); +} +``` + +## GitlabInstance + +The `GitlabInstance` class was created in order to manage the APIs and +information related to the GitLab profile, the project, and the user-specific +data stored in their account. + +The username and the token required to instantiate the Gitbeaker _Gitlab_ +component, which is required for making the API calls, are retrieved from the +session storage, taking the _access_token_ of the user already logged into the +DTaaS application. + +The initialisation of the `GitlabInstance` object is concluded with the +execution of the `init()` method, which enables the retrieval and storage of +the `projectId` and `triggerToken` attributes. The _projectId_ is a unique +identifier for projects in GitLab and it is essential for subsequent API calls. +For example, it is passed to the method that retrieves a _trigger token_, +which is used to trigger CI/CD pipelines in GitLab. + +The objective of the `getDTSubfolders` method was to retrieve the names and +corresponding descriptions of the DTs of the user, so that these could be shown +at the front-end interface. This approach would obviate the user from having to +input the name of a DT; hence, saving the user from possible error and +inefficiencies arising from manual input. The user interface makes it easier +for the user to deal with DTs by automatizing their selection and manages them +more accurately. This implementation also eliminates the necessity for manual +input from users for the access token and the username, which are automatically +provided via the GitLab OAuth login. + +Furthermore, logs maintained in the `GitlabInstance` class improve awareness +and transparency over the operations conducted. The final three methods are +employed in conjunction to oversee the execution of a DT. In particular, +individual logs are saved for each job in the pipeline, and the status of the +latter is monitored so that, once the entire pipeline is complete, the results +can be displayed in detail within the user interface. In this way, all statuses +of each operation are logged for better debugging and performance analysis, +including possible errors. Having trace logs exposed to the user means +troubleshooting will be more effective and insight into execution and +management of DTs will be gained, improving system reliability and user +confidence. + +## DigitalTwin + +The DigitalTwin class was created in order to manage the APIs and information +related to a specific DT. + +The creation of a DigitalTwin object requires a pre-existing GitlabInstance to +be associated with the object. It was determined that matching a different +GitlabInstance for each DigitalTwin would be the optimal approach to ensure the +maintenance of independence between the various DTs. The _api_ attribute of +GitlabInstance facilitates the execution of Gitbeaker APIs pertinent to the DT. + +The class allows a pipeline to be started and stopped, thus giving the user full +control of the execution. The execute() method uses the previous methods +internally. This approach ensures that there are no errors due to missing design +information during the execution of the pipeline. Responsibilities have been +divided into smaller methods in order to make the code more modular, facilitating +debugging and testing. In both execute() and stop(), the status of operations +executed on the DT is monitored, keeping track of them via the logs attribute of +GitlabInstance. Errors are identified and tracked, providing a complete view and +the ability to monitor performance. + +The `descriptionFiles`, `lifecycleFiles` and `configFiles` attributes are used +to keep track of the files within the corresponding GitLab folder of the +DT, thus enabling the read and modify features.
+The create() method enables the creation of a DT and saves all its files +in the user’s corresponding GitLab folder. Additionally, if the DT is +configured as _common_, it is also added to GitLab’s shared folder, +making it part of the Library and accessible to other users. + +Similarly, the delete() method removes a DT from GitLab. If the DT was +part of the Library, it is also removed from the shared folder. + +A crucial aspect of these two methods is their integration with the +DevOps infrastructure. When a DT is created or deleted, the +`.gitlab-ci.yml` file of the parent pipeline is updated to add or remove +the _trigger_DTName_ section associated with the DT. This ensures that a +user-created DT can be executed via the web interface without requiring +manual updates to pipeline configuration files on GitLab. Instead, these +files are automatically updated, providing an effortless user experience +and maintaining alignment with the infrastructure. + +## LibraryAsset + +The LibraryAsset class was created in order to manage the APIs and information +related to a specific library asset. + +It is similar to the DigitalTwin class, but contains only the methods required +to display files. This focused design reflects its limited scope and ensures +simplicity and clarity for use cases involving the library. + +--- + +Ref: Vanessa Scherma, Design and implementation of an integrated DevOps +framework for Digital Twins as a Service software platform, +Master's Degree Thesis, Politecnico Di Torino, 2024. diff --git a/docs/developer/devops/overview.md b/docs/developer/devops/overview.md new file mode 100644 index 000000000..d48c226a4 --- /dev/null +++ b/docs/developer/devops/overview.md @@ -0,0 +1,94 @@ +# Overview + +## Expectations From a DevOps Framework + +The functional requirements of the system include the automation of pipelines +and the management of Digital Twins (DTs) via Application Programming Interface +(API). Consequently, the framework was designed to facilitate the comprehensive +automation of the DT lifecycle, minimizing the necessity for manual +intervention. The system must be capable of managing the dynamic configuration +of pipelines, utilizing variables that permit the customization of pipeline +behavior according to the data provided by the user, such as the designation of +the DT. + +Integration with GitLab is another fundamental requirement. The framework must +be able to interact with GitLab to execute CI/CD pipelines via API calls using +Gitbeaker as a wrapper. Users must authenticate via GitLab's OAuth mechanism, +and the system must automatically manage the authentication tokens and trigger +tokens needed to start pipelines. Additionally, the system must automatically +retrieve key information from the user's GitLab repository, such as the list of +available DTs. + +## High Level Architecture + +We use a DevOps framework to enable interaction with the DTs via APIs calls, so +that users can start, monitor and manage their DTs via the web +application. + +The architectural design of the DevOps framework was intended to facilitate the +management of DTs. It is based on two key elements: + +- __The GitLab CI/CD infrastructure__, which employs a parent-child pipeline + hierarchy. The objective of this infrastructure is to enable the triggering + of a pipeline of a specific DT by simply passing the necessary data as + parameters, such as the name of the DT and the tag of the runner that will + execute the pipeline. + +- __Classes implemented in the code__, which utilize Gitbeaker to realize the + APIs required for interaction with DTs. + +The component diagram below illustrates how the infrastructure consists of three +main classes: `DigitalTwin`, `LibraryAsset`, and `GitlabInstance`. + +![High-level design of the DevOps framework](./images/devops-overview.png) + +The distinction between the `DigitalTwin` and `LibraryAsset` classes was +necessary to separate the full management of a DT from an asset visualized +through the library. The `LibraryAsset` class provides a significantly reduced +set of functionality compared to the `DigitalTwin`, focusing only on asset +visualization. + +Intermediate classes were introduced to ensure a clear separation of file +management responsibilities: `DTAssets` and `LibraryManager`. These classes +implement the necessary logic to mediate between a `DigitalTwin` or +`LibraryAsset` and the `FileHandler` class. The `FileHandler` class has a +single responsibility: to make API calls to files via GitBeaker. This design +allows for the separation of high-level logic from low-level file operations. + +The infrastructure requires that the `DigitalTwin` class and the `LibraryAsset` +class include an instance of `GitlabInstance`. This composition relationship +emphasizes the dependency between these classes, where a `DigitalTwin` or a +`LibraryAsset` instance cannot function independently without a `GitlabInstance`. +The `GitlabInstance` class provides the essential services required for +interacting with GitLab, including API integrations and pipeline management. + +The `GitlabInstance` class serves as the interface to the realized CI/CD +infrastructure. By utilizing the GitLab class imported from GitBeaker and +initialized as its attribute, `GitlabInstance` facilitates the execution of +pipelines and other CI/CD-related tasks. This architecture ensures that the +infrastructure remains modular and adheres to the principles of single +responsibility and clear dependency management. + +## Gitbeaker + +[GitBeaker is a client library for Node.js](https://github.com/jdalrymple/gitbeaker) +that enables users to interact with the GitLab API. In particular, +`gitbeaker/rest` is a specific version of the Gitbeaker package that allows +users to submit requests to GitLab's REST API. + +One of the most significant features of Gitbeaker is the provision of support +for a range of authentication methods, including the use of personal tokens +and OAuth keys. Gitbeaker provides a range of predefined methods for +requesting data from the various GitLab APIs, eliminating the need for users +to manually construct HTTP requests, thus greatly simplifying the integration +process with GitLab. + +It automatically handles errors in HTTP requests (and provides meaningful error +messages that help diagnose and resolve problems) and is fully compatible with +all of GitLab's REST APIs. + +--- + +Ref: Vanessa Scherma, Design and implementation of an integrated DevOps +framework for Digital Twins as a Service software platform, +Master's Degree Thesis, Politecnico Di Torino, 2024. diff --git a/docs/developer/index.md b/docs/developer/index.md index 0b6354ba9..49e6d29ae 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -1,6 +1,17 @@ -# Developer Guide +# Contributors Guide + +Welcome to the Digital Twin as a Service (DTaaS) contributing guide + +Thank you for investing your time in contributing to our project! + +Read our [Code of Conduct](conduct.md) to keep our community +approachable and respectable. + +In this guide you will get an overview of the contribution workflow +from opening an issue, creating a PR, reviewing, and merging the PR. + +## Project Goals -This guide is for DTaaS platform developers. It helps development team members get familiar with the DTaaS project software design, and development processes. Please see developer-specific @@ -10,15 +21,26 @@ and [Research paper](https://arxiv.org/abs/2305.07244). ## :computer: Development Environment -Ideally, developers should work on Ubuntu/Linux Operating System.There is -an ongoing effort -to bring support for Windows Operating System. But, the development and -software installation scripts are still heavily suited to Ubuntu/Linux +Please use the steps given here to have suitable development +environment. -Please use the steps given here to install the required software packages. +### DevContainers + +There is a [devcontainer configuration](.devcontainer/devcontainer.json) +for the project. Please use it to get a dockerized development environment. +DevContainer is the easiest way to get started. ### Ubuntu/Linux +The code base has been developed for most part on +Ubuntu/Linux Operating System.Thus certain parts of the code base might +have bugs when run on Windows. At the moment, only +[runner](./servers/execution/runner/DEVELOPER.md) has problems running +on non-Linux OS. + +The development environment can be installed by using the following +scripts. + ```bash bash script/env.sh bash script/docker.sh @@ -30,6 +52,11 @@ You will have to download the docker images on a really good network. ### Windows +The development environment scripts for Windows are still buggy. +Any help in improving them is greatly appreciated. +Given that, caveat, please use the following installation steps +for Windows. + Two powershell installation scripts, namely `base.ps1` and `env.ps1` are available to install the required software packages. But errors might crop up due to missing @@ -94,9 +121,6 @@ a development workflow is in place. Each developer should follow these steps: branch to your `feature/distributed-demo` branch. The PR will run all the github actions, code climate and codecov checks. 1. Resolve all the issues identified in the previous step. -1. If you have access to the - [integration server](https://github.com/INTO-CPS-Association/DTaaS/wiki/DTaaS-Integration-Server), - try your working branch on the integration server. 1. Once changes are verified, a PR should be made to the `feature/distributed-demo` branch of the upstream diff --git a/docs/user/digital-twins/devops/dt_execute_log.png b/docs/user/digital-twins/devops/dt_execute_log.png new file mode 100644 index 000000000..c69a8acaa Binary files /dev/null and b/docs/user/digital-twins/devops/dt_execute_log.png differ diff --git a/docs/user/digital-twins/devops/dt_execute_running.png b/docs/user/digital-twins/devops/dt_execute_running.png new file mode 100644 index 000000000..7cab8657c Binary files /dev/null and b/docs/user/digital-twins/devops/dt_execute_running.png differ diff --git a/docs/user/digital-twins/devops/dt_manage.png b/docs/user/digital-twins/devops/dt_manage.png new file mode 100644 index 000000000..1162347ba Binary files /dev/null and b/docs/user/digital-twins/devops/dt_manage.png differ diff --git a/docs/user/digital-twins/devops/dt_manage_details.png b/docs/user/digital-twins/devops/dt_manage_details.png new file mode 100644 index 000000000..2212342d6 Binary files /dev/null and b/docs/user/digital-twins/devops/dt_manage_details.png differ diff --git a/docs/user/digital-twins/devops/dt_manage_reconfigure.png b/docs/user/digital-twins/devops/dt_manage_reconfigure.png new file mode 100644 index 000000000..a4dc85830 Binary files /dev/null and b/docs/user/digital-twins/devops/dt_manage_reconfigure.png differ diff --git a/docs/user/digital-twins/devops/file-structure.md b/docs/user/digital-twins/devops/file-structure.md new file mode 100644 index 000000000..235fca949 --- /dev/null +++ b/docs/user/digital-twins/devops/file-structure.md @@ -0,0 +1,65 @@ +# Digital Twin File Structure in Gitlab + +We use GitLab as a file store for performing DevOps on Digital Twins. +The [user interface page](ui.md) is a front-end for this gitlab-backed +file storage. + +Each DTaaS installation comes with an integrated Gitlab. There must +be a gitlab group named **dtaas** and a gitlab repository for each +user where repository name matches the username. For example, +if there are two users, namely _user1_ and _user2_ on a DTaaS +installation, then the following repositories must exist on the linked +Gitlab installation. + +```txt +https://foo.com/gitlab/dtaas/common.git +https://foo.com/gitlab/dtaas/user1.git +https://foo.com/gitlab/dtaas/user2.git +``` + +Each user repository must also have a specific structure. The required structure +is as follows. + +```text +/ +├── common/ +├── data/ +├── digital_twins/ +├── functions/ +├── models/ +├── tools/ +├── .gitlab-ci.yml +└── README.md +``` + +This file structure follows the same pattern user sees on the existing +**Library** page. + +## Digital Twin Structure + +The `digital_twins` folder contains DTs that have been pre-built by one or +more users. The intention is that they should be sufficiently flexible to be +reconfigured as required for specific use cases. + +Let us look at an example of such a configuration. The +[dtaas/user1 repository on gitlab.com](https://gitlab.com/dtaas/user1) contains +the `digital_twins` directory with a `hello_world` example. Its file structure +looks like this: + +```text +hello_world/ +├── lifecycle/ +│ ├── clean +│ ├── create +│ ├── execute +│ └── terminate +├── .gitlab-ci.yml +└── description.md +``` + +The `lifecycle` directory contains four files - `clean`, `create`, `execute` +and `terminate`, which are executable scripts. The `hello-world` example uses +simple [BASH scripts](https://www.gnu.org/software/bash/). +These correspond to stages in a digital twin's lifecycle. +Further explanation of digital twin is available on +[lifecycle stages](../lifecycle.md). diff --git a/docs/user/digital-twins/devops.md b/docs/user/digital-twins/devops/ui.md similarity index 89% rename from docs/user/digital-twins/devops.md rename to docs/user/digital-twins/devops/ui.md index 3bcb68156..4ba7ec2f6 100644 --- a/docs/user/digital-twins/devops.md +++ b/docs/user/digital-twins/devops/ui.md @@ -15,14 +15,14 @@ in the DTaaS. This feature requires specific installation setup. -1. [Integrated gitlab installation](../../admin/gitlab/integration.md) +1. [Integrated gitlab installation](../../../admin/gitlab/integration.md) 1. A valid gitlab repository for the logged in user. Please see an [example repository](https://gitlab.com/dtaas/user1). You can clone this repository and customize to your needs. -1. [A linked Gitlab Runner](../../admin/gitlab/runner.md) +1. [A linked Gitlab Runner](../../../admin/gitlab/runner.md) to the user gitlab repository. -## Manage +## Manage Tab ![Manage Digital Twin Preview Page](dt_manage.png) @@ -34,7 +34,7 @@ The manage tab allows for different operations on a digital twin: A digital twin placed in the DTaaS has a certain recommended structure. Please see the -[assets pag](../servers/lib/assets.md) for an explanation +[assets pag](../../servers/lib/assets.md) for an explanation and [this example](https://github.com/INTO-CPS-Association/DTaaS-examples/tree/main/digital_twins/mass-spring-damper). @@ -53,7 +53,7 @@ correspond to three categories. ![Digital Twin Reconfigure](dt_manage_reconfigure.png) -## Execute +## Execute Tab ![Digital Twin Execute](dt_execute_running.png) diff --git a/mkdocs-github.yml b/mkdocs-github.yml index b20988935..b3fed630a 100644 --- a/mkdocs-github.yml +++ b/mkdocs-github.yml @@ -26,7 +26,9 @@ nav: - Digital Twins: - Create: user/digital-twins/create.md - Lifecycle: user/digital-twins/lifecycle.md - - DevOps Preview: user/digital-twins/devops.md + - DevOps Preview: + - Digital Twin Structure: user/digital-twins/devops/file-structure.md + - User Interface: user/digital-twins/devops/ui.md - Runner: user/servers/execution/runner/README.md - Examples: - Overview: user/examples/index.md @@ -85,6 +87,7 @@ nav: - FAQ: FAQ.md - Developer: - Developer Guide: developer/index.md + - Code of Conduct: developer/conduct.md - GitHub Actions: developer/github-actions.md - System: - Architecture: developer/system/architecture.md @@ -96,6 +99,11 @@ nav: - Testing: developer/testing/intro.md - Docker: developer/docker/docker.md - NPM Packages: developer/npm-packages.md + - DevOps Framework: + - Overview: developer/devops/overview.md + - GitLab CI/CD Infrastructure: developer/devops/gitlab-ci-cd.md + - Triggering Pipelines via API: developer/devops/api.md + - Implemented Classes: developer/devops/implementation.md - Bugs: bugs.md - Thanks: thanks.md - License: diff --git a/mkdocs.yml b/mkdocs.yml index 77c0c4c3f..5cd43ec3f 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,7 +26,9 @@ nav: - Digital Twins: - Create: user/digital-twins/create.md - Lifecycle: user/digital-twins/lifecycle.md - - DevOps Preview: user/digital-twins/devops.md + - DevOps Preview: + - Digital Twin Structure: user/digital-twins/devops/file-structure.md + - User Interface: user/digital-twins/devops/ui.md - Runner: user/servers/execution/runner/README.md - Examples: - Overview: user/examples/index.md @@ -85,6 +87,7 @@ nav: - FAQ: FAQ.md - Developer: - Developer Guide: developer/index.md + - Code of Conduct: developer/conduct.md - GitHub Actions: developer/github-actions.md - System: - Architecture: developer/system/architecture.md @@ -96,6 +99,11 @@ nav: - Testing: developer/testing/intro.md - Docker: developer/docker/docker.md - NPM Packages: developer/npm-packages.md + - DevOps Framework: + - Overview: developer/devops/overview.md + - GitLab CI/CD Infrastructure: developer/devops/gitlab-ci-cd.md + - Triggering Pipelines via API: developer/devops/api.md + - Implemented Classes: developer/devops/implementation.md - Bugs: bugs.md - Thanks: thanks.md - License: diff --git a/servers/execution/runner/yarn.lock b/servers/execution/runner/yarn.lock index a906980ea..3c2382340 100644 --- a/servers/execution/runner/yarn.lock +++ b/servers/execution/runner/yarn.lock @@ -393,9 +393,9 @@ integrity sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ== "@eslint/plugin-kit@^0.2.0": - version "0.2.1" - resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.2.1.tgz#cd14fe2db79fa639839dfef4105e83bad1814482" - integrity sha512-HFZ4Mp26nbWk9d/BpvP0YNL6W4UoZF0VFcTw/aPPA8RpOxeFQgK+ClABGgAUXs9Y/RGX/l1vOmrqz1MQt9MNuw== + version "0.2.4" + resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.2.4.tgz#2b78e7bb3755784bb13faa8932a1d994d6537792" + integrity sha512-zSkKow6H5Kdm0ZUQUB2kV5JIXqoG0+uH5YADhaEHswm664N9Db8dXSi0nMJpacpMf+MyyglF1vnZohpEg5yUtg== dependencies: levn "^0.4.1"