From c9f1755eb5c1e6aec27d6c15f44830d86b59c6ff Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 16 Mar 2023 09:25:44 -0400 Subject: [PATCH 1/7] start --- .travis.yml | 73 ------------------- ...ravis_centaur.md => centaur_test_types.md} | 19 +---- 2 files changed, 4 insertions(+), 88 deletions(-) delete mode 100644 .travis.yml rename docs/developers/bitesize/ci/{travis_centaur.md => centaur_test_types.md} (83%) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 295d133cbaa..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,73 +0,0 @@ -os: linux -dist: focal -services: - - docker -language: minimal -git: - depth: false -cache: - directories: - - $HOME/.ivy2/cache - - $HOME/.coursier/cache - # see cromwell::private::delete_sbt_boot for more info - #- $HOME/.sbt/boot/ -before_cache: - # Tricks to avoid unnecessary cache updates - - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete - - find $HOME/.coursier/cache -name "ivydata-*.properties" -print -delete - - find $HOME/.sbt -name "*.lock" -print -delete -env: - jobs: - # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked - - >- - BUILD_TYPE=centaurAws - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurDummy - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurEngineUpgradeLocal - BUILD_MYSQL=5.7 - # Temporarily keeping until `centaurEngineUpgradePapiV2beta` or similar exists - - >- - BUILD_TYPE=centaurEngineUpgradePapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurHoricromtalPapiV2beta - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurLocal - BUILD_MARIADB=10.3 - - >- - BUILD_TYPE=centaurLocal - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurLocal - BUILD_POSTGRESQL=11.3 - - >- - BUILD_TYPE=centaurPapiV2beta - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurTes - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=checkPublish - - >- - BUILD_TYPE=horicromtalDeadlock - - >- - BUILD_TYPE=dbms - - >- - BUILD_TYPE=singleWorkflowRunner - - >- - BUILD_TYPE=referenceDiskManifestBuilderApp -script: - - src/ci/bin/test.sh -notifications: - slack: - rooms: - - secure: B5KYcnhk/ujAUWlHsjzP7ROLm6MtYhaGikdYf6JYINovhMbVKnZCTlZEy7rqT3L2T5uJ25iefD500VQGk1Gn7puQ1sNq50wqjzQaj20PWEiBwoWalcV/nKBcQx1TyFT13LJv8fbFnVPxFCkC3YXoHedx8qAhDs8GH/tT5J8XOC8= - template: - - "Build <%{build_url}|#%{build_number}> (<%{compare_url}|%{commit}>) of %{repository}@%{branch} by %{author} %{result} in %{duration}" - on_success: change - on_failure: change - on_pull_requests: false diff --git a/docs/developers/bitesize/ci/travis_centaur.md b/docs/developers/bitesize/ci/centaur_test_types.md similarity index 83% rename from docs/developers/bitesize/ci/travis_centaur.md rename to docs/developers/bitesize/ci/centaur_test_types.md index b56ec16c8af..14883eb3b9c 100644 --- a/docs/developers/bitesize/ci/travis_centaur.md +++ b/docs/developers/bitesize/ci/centaur_test_types.md @@ -1,17 +1,4 @@ -## Travis builds by user access - -For infrastructures that require secured credentials, cloud backend tests only run for developers with write access to the broadinstitute/cromwell GitHub. Secure tests are skipped for all other users. - -Other backends run tests for any user. - -| Backend | Read-only users | Write/Admin users | -|---------------|:---------------:|:-----------------:| -| AWS | | ✅ | -| Local | ✅ | ✅ | -| PAPI V2alpha1 | | ✅ | -| PAPI V2beta | | ✅ | -| SLURM | ✅ | ✅ | -| TES | ✅ | ✅ | +## Centaur Test Types ## Upgrade / Horicromtal / etc. @@ -20,8 +7,9 @@ Other backends run tests for any user. | Engine Upgrade | `(backend)_application.conf` | `centaur_application.conf`* | | Horicromtal | `papi_[v2beta or v2alpha1]_horicromtal_application.conf`** | `centaur_application_`
`horicromtal.conf` | | Horicromtal
Engine Upgrade | `papi_v2beta_application.conf`** | `centaur_application_`
`horicromtal_no_assert.conf` | -| Papi Upgrade | `papi_v1_v2alpha1_upgrade_application.conf`** | `centaur_application.conf`* | +| PAPI Upgrade | `papi_v1_v2alpha1_upgrade_application.conf`** | `centaur_application.conf`* | | Papi Upgrade
New Workflows | `(backend)_application.conf` | `centaur_application.conf`* | +| Azure Blob | `centaur_blob_test.conf`** | `centaur_application.conf`* | | WDL Upgrade | `(backend)_application.conf` | `centaur_application.conf`* | | (other) | `(backend)_application.conf` | `centaur_application.conf`* | @@ -32,6 +20,7 @@ Other backends run tests for any user. | Horicromtal
Engine Upgrade | `EngineUpgradeTestCaseSpec` | `engineUpgradeTestCases`*** | | PAPI Upgrade | `PapiUpgradeTestCaseSpec` | `papiUpgradeTestCases` | | PAPI Upgrade
New Workflows | `CentaurTestSuite` | `papiUpgradeNewWorkflowsTestCases` | +| Azure Blob | `CentaurTestSuite ` | `azureBlobTestCases` | | WDL Upgrade | `WdlUpgradeTestCaseSpec` | `standardTestCases`**** | | (other) | `CentaurTestSuite` | `standardTestCases` | From dc3d197ef133e90cc6251f04749c6621b1fff23d Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 23 Mar 2023 11:23:50 -0400 Subject: [PATCH 2/7] more cleanup --- docs/developers/bitesize/ci/CICD.md | 20 ++------------------ mkdocs.yml | 1 - 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/developers/bitesize/ci/CICD.md b/docs/developers/bitesize/ci/CICD.md index 1353deb8f91..8465a079044 100644 --- a/docs/developers/bitesize/ci/CICD.md +++ b/docs/developers/bitesize/ci/CICD.md @@ -1,24 +1,8 @@ Cromwell uses [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_integration) (CI) testing, along with [Continuous Delivery](https://en.wikipedia.org/wiki/Continuous_delivery) (CD) to the Cromwell-as-a-Service (CaaS) `DEV` environment. [Continuous Deployment](https://en.wikipedia.org/wiki/Continuous_deployment) is not implemented. -## CI testing in Travis and Jenkins +## CI Testing in Github Actions -Any suite of tests running under 2.5 hours, using 2 cpus, and 6gb of memory executes on [Travis](https://travis-ci.com/broadinstitute/cromwell/). Travis tests every pull request by a trusted contributor. Larger test suites run on [Jenkins instances](https://github.com/broadinstitute/dsp-jenkins#readme). Examples include the DSP Workbench CI testing ([swatomation](https://fc-jenkins.dsp-techops.broadinstitute.org/job/swatomation-pipeline/)), a nightly test of `develop` on (a snapshot of) the [$5 Genome WDL](https://fc-jenkins.dsp-techops.broadinstitute.org/job/cromwell-cron-aws/), and the [Cromwell-Perf tests](https://fc-jenkins.dsp-techops.broadinstitute.org/job/cromwell-perf-cron/) that call-cache thousands of jobs. - -## CD to CaaS DEV - -![CaaS DEV CD](CaaS_DEV_CD.svg) - -One instance utilizing continuous delivery is the `develop` branch to CaaS `DEV`. While manual testing could occur on `DEV`, users primarily test on `PROD`. - -## Manual Deployments - -![Cromwell Deployment Strategies](Cromwell_Deployment_Strategies.svg) - -The Cromwell developers stage Terra and CaaS `PROD` deployments. All other deployments are performed by respective groups, who decide when and how to redeploy. Some upgrade Cromwell, while others deploy completely new instances, including a new database schema. - -## Code Coverage - -Only Travis Pull Requests generate [maximal code coverage reports](https://codecov.io/gh/broadinstitute/cromwell/pulls). All other CI either doesn't report coverage, or under-reports due to skipped tests. +All CI testing of Cromwell is done through [Github Actions](https://github.com/broadinstitute/cromwell/actions). Github Actions will test every pull request submitted by a trusted contributor. Pull requests must pass all unit and integration tests in order to be merged into the develop environment. ## Vulnerability Scanning diff --git a/mkdocs.yml b/mkdocs.yml index 7aea4302d34..802b6e86196 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -103,7 +103,6 @@ nav: - General Topics: - Simpletons: developers/bitesize/general/simpletons.md - Continuous Integration (CI) / Continuous Delivery (CD): developers/bitesize/ci/CICD.md - - Travis Centaur: developers/bitesize/ci/travis_centaur.md theme: readthedocs From dafe0c66dd0828b8ca8d1198c96429122ec0b382 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 23 Mar 2023 13:18:44 -0400 Subject: [PATCH 3/7] more cleanup --- docs/developers/Centaur.md | 74 ++++++++++++++++++- docs/developers/Contribute.md | 7 -- docs/developers/bitesize/ci/CICD.md | 13 ---- docs/developers/bitesize/ci/CaaS_DEV_CD.svg | 1 - .../ci/Cromwell_Deployment_Strategies.svg | 1 - .../bitesize/ci/centaur_test_types.md | 64 ---------------- mkdocs.yml | 2 +- 7 files changed, 73 insertions(+), 89 deletions(-) delete mode 100644 docs/developers/bitesize/ci/CICD.md delete mode 100644 docs/developers/bitesize/ci/CaaS_DEV_CD.svg delete mode 100644 docs/developers/bitesize/ci/Cromwell_Deployment_Strategies.svg delete mode 100644 docs/developers/bitesize/ci/centaur_test_types.md diff --git a/docs/developers/Centaur.md b/docs/developers/Centaur.md index cf168a3026c..35d2e350e7d 100644 --- a/docs/developers/Centaur.md +++ b/docs/developers/Centaur.md @@ -14,9 +14,11 @@ You can now run the tests from another terminal. There are two ways to invoke the integration tests: -* `sbt "centaur / IntegrationTest / test"` - compiles and run via sbt directly, simple but also has the problem of running 2x cores tests in parallel which can overwhelm your Cromwell server if running in a development environment +* `sbt "centaur / IntegrationTest / test"` - compiles Centaur and runs all tests via sbt directly. Tests are expected to be in the `centaur/src/main/standardTestCases` directory. This can be changed by modifying `reference.conf`. -* `src/ci/bin/testCentaurLocal.sh` - runs the same tests using the continuous integration pipeline configuration +* `src/ci/bin/testCentaurLocal.sh` - runs the same tests using the continuous integration pipeline configuration. + +* Tests that require different Cromwell and Centaur configurations can be invoked by calling the various scripts in `src/ci/bin`. ### Tags @@ -104,3 +106,71 @@ In case the absolute path the cromwell root is used (for example: `/home/my_user In case testing of the caching is required `<>` can be used. The testFormat should be `runtwiceexpectingcallcaching`. + + +## Centaur Test Types +Both Cromwell and Centaur require configuration files in order to correctly build and test various parts of the system. Because of this, we divide +our tests into groups depending on which configuration files they require. Below is the current matrix of configuration files and test source directories. + +## Upgrade / Horicromtal / etc. + +| CI Test Type | Cromwell Config | Centaur Config | +|-------------------------------|------------------------------------------------------------------|--------------------------------------------------------| +| Engine Upgrade | `(backend)_application.conf` | `centaur_application.conf`* | +| Horicromtal | `papi_[v2beta or v2alpha1]_horicromtal_application.conf`** | `centaur_application_`
`horicromtal.conf` | +| Horicromtal
Engine Upgrade | `papi_v2beta_application.conf`** | `centaur_application_`
`horicromtal_no_assert.conf` | +| PAPI Upgrade | `papi_v1_v2alpha1_upgrade_application.conf`** | `centaur_application.conf`* | +| Papi Upgrade
New Workflows | `(backend)_application.conf` | `centaur_application.conf`* | +| Azure Blob | `centaur_blob_test.conf`** | `centaur_application.conf`* | +| WDL Upgrade | `(backend)_application.conf` | `centaur_application.conf`* | +| (other) | `(backend)_application.conf` | `centaur_application.conf`* | + +| CI Test Type | ScalaTest Spec | Test Directory | +|-------------------------------|-----------------------------|-------------------------------------| +| Engine Upgrade | `EngineUpgradeTestCaseSpec` | `engineUpgradeTestCases` | +| Horicromtal | `CentaurTestSuite` | `standardTestCases`*** | +| Horicromtal
Engine Upgrade | `EngineUpgradeTestCaseSpec` | `engineUpgradeTestCases`*** | +| PAPI Upgrade | `PapiUpgradeTestCaseSpec` | `papiUpgradeTestCases` | +| PAPI Upgrade
New Workflows | `CentaurTestSuite` | `papiUpgradeNewWorkflowsTestCases` | +| Azure Blob | `CentaurTestSuite ` | `azureBlobTestCases` | +| WDL Upgrade | `WdlUpgradeTestCaseSpec` | `standardTestCases`**** | +| (other) | `CentaurTestSuite` | `standardTestCases` | + + +\* Centaur Config always uses `centaur_application.conf` except when overridden with `papi_v2alpha1_centaur_application.conf` +or `papi_v2beta_centaur_application.conf` + ([48 preview link](https://github.com/broadinstitute/cromwell/blob/a7d0601/src/ci/bin/test.inc.sh#L455-L457)) +\*\* Cromwell Config overrides + ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L213-L221)) +\*\*\* Test Directory overrides + ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L440-L449)) +\*\*\*\* Test Directory only tests tagged with `wdl_upgrade` + ([47 link](https://github.com/broadinstitute/cromwell/blob/47/centaur/src/main/resources/standardTestCases/write_lines.test#L3)) + + +- Engine Upgrade: Retrieves the [Cromwell Version](https://github.com/broadinstitute/cromwell/blob/47/project/Version.scala#L8) then retrieves the previous jar/docker-image from DockerHub. Centaur starts with the prior version, then restarts with the compiled source code. +- Horicromtal: Runs a [docker-compose](https://github.com/broadinstitute/cromwell/blob/47/src/ci/docker-compose/docker-compose-horicromtal.yml) with: + 1. db-mstr: started first + 2. sum-back: runs summarizer + 3. front-back: exposes HTTP +- Horicromtal Engine Upgrade: Combination of Horicromtal and Engine Upgrade +- PAPI Upgrade: Tests run with an older version of Papi and upon restart use a newer version of Papi +- PAPI Upgrade New Workflows: Test definition [does not run any tests](https://travis-ci.org/broadinstitute/cromwell/jobs/475378412) +- WDL Upgrade: Upgrades WDL from draft-2 to 1.0 before testing +- (other): Runs `*.test` files listing the configured backend names + +## RDBMS + +| Backend | MySQL | PostgreSQL | MariaDB | +|---------|:------:|:-----------:|:--------:| +| AWS | ✅ | | | +| Local | ✅ | ✅ | | +| PAPI V2 | ✅ | | ⭕ | +| SLURM | ✅ | | | +| TES | ✅ | | | + + +⭕ Tests Horicromtal Engine Upgrade versus standard Centaur suite + + +All backends run against MySQL. The Local backend also test PostgreSQL, allowing contributors ensure WDLs work with PostgreSQL. MariaDB is tested on a specialized upgrade, where the MySQL connector client is used first, and the MariaDB client is used after restart. diff --git a/docs/developers/Contribute.md b/docs/developers/Contribute.md index 1f513bd264e..9f40fb982ee 100644 --- a/docs/developers/Contribute.md +++ b/docs/developers/Contribute.md @@ -8,13 +8,6 @@ First off, here are useful links: * Source on Github: [github.com/broadinstitute/cromwell](https://github.com/broadinstitute/cromwell/tree/develop/docs) * Builds on ReadTheDocs: [readthedocs.org/projects/cromwell](https://readthedocs.org/projects/cromwell/builds/) * How to build and view the documentation locally: [mkdocs.readthedocs.io](https://mkdocs.readthedocs.io/en/stable/#installation) -* Public Google Docs - Table of Contents: [drive.google.com/open?id=1myTVzWx5HG720nPBUAF9vE8XTacrzCW70RpgsjMnchM](https://drive.google.com/open?id=1myTVzWx5HG720nPBUAF9vE8XTacrzCW70RpgsjMnchM) - -### "needs docs" - -There are plenty of areas of the Cromwell documentation that need to be updated, improved, or added. Within the Cromwell repo on Github there are many issues labeled as ["needs docs"](https://github.com/broadinstitute/cromwell/issues?q=is%3Aopen+is%3Aissue+label%3A%22needs+docs%22), so feel free to start there. - -If you would like to request additional documentation, you can create a [Github issue in the Cromwell repo](https://github.com/broadinstitute/cromwell/issues/new). ### Writing Tips diff --git a/docs/developers/bitesize/ci/CICD.md b/docs/developers/bitesize/ci/CICD.md deleted file mode 100644 index 8465a079044..00000000000 --- a/docs/developers/bitesize/ci/CICD.md +++ /dev/null @@ -1,13 +0,0 @@ -Cromwell uses [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_integration) (CI) testing, along with [Continuous Delivery](https://en.wikipedia.org/wiki/Continuous_delivery) (CD) to the Cromwell-as-a-Service (CaaS) `DEV` environment. [Continuous Deployment](https://en.wikipedia.org/wiki/Continuous_deployment) is not implemented. - -## CI Testing in Github Actions - -All CI testing of Cromwell is done through [Github Actions](https://github.com/broadinstitute/cromwell/actions). Github Actions will test every pull request submitted by a trusted contributor. Pull requests must pass all unit and integration tests in order to be merged into the develop environment. - -## Vulnerability Scanning - -In collaboration with the DSP Information Security team, scans include but are not limited to: - -- Committed Git secrets -- Vulnerable Java dependencies -- Penetration testing diff --git a/docs/developers/bitesize/ci/CaaS_DEV_CD.svg b/docs/developers/bitesize/ci/CaaS_DEV_CD.svg deleted file mode 100644 index 9b91e73f87a..00000000000 --- a/docs/developers/bitesize/ci/CaaS_DEV_CD.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/developers/bitesize/ci/Cromwell_Deployment_Strategies.svg b/docs/developers/bitesize/ci/Cromwell_Deployment_Strategies.svg deleted file mode 100644 index e911f3a6e98..00000000000 --- a/docs/developers/bitesize/ci/Cromwell_Deployment_Strategies.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/developers/bitesize/ci/centaur_test_types.md b/docs/developers/bitesize/ci/centaur_test_types.md deleted file mode 100644 index 14883eb3b9c..00000000000 --- a/docs/developers/bitesize/ci/centaur_test_types.md +++ /dev/null @@ -1,64 +0,0 @@ -## Centaur Test Types - -## Upgrade / Horicromtal / etc. - -| CI Test Type | Cromwell Config | Centaur Config | -|-------------------------------|------------------------------------------------------------------|--------------------------------------------------------| -| Engine Upgrade | `(backend)_application.conf` | `centaur_application.conf`* | -| Horicromtal | `papi_[v2beta or v2alpha1]_horicromtal_application.conf`** | `centaur_application_`
`horicromtal.conf` | -| Horicromtal
Engine Upgrade | `papi_v2beta_application.conf`** | `centaur_application_`
`horicromtal_no_assert.conf` | -| PAPI Upgrade | `papi_v1_v2alpha1_upgrade_application.conf`** | `centaur_application.conf`* | -| Papi Upgrade
New Workflows | `(backend)_application.conf` | `centaur_application.conf`* | -| Azure Blob | `centaur_blob_test.conf`** | `centaur_application.conf`* | -| WDL Upgrade | `(backend)_application.conf` | `centaur_application.conf`* | -| (other) | `(backend)_application.conf` | `centaur_application.conf`* | - -| CI Test Type | ScalaTest Spec | Test Directory | -|-------------------------------|-----------------------------|-------------------------------------| -| Engine Upgrade | `EngineUpgradeTestCaseSpec` | `engineUpgradeTestCases` | -| Horicromtal | `CentaurTestSuite` | `standardTestCases`*** | -| Horicromtal
Engine Upgrade | `EngineUpgradeTestCaseSpec` | `engineUpgradeTestCases`*** | -| PAPI Upgrade | `PapiUpgradeTestCaseSpec` | `papiUpgradeTestCases` | -| PAPI Upgrade
New Workflows | `CentaurTestSuite` | `papiUpgradeNewWorkflowsTestCases` | -| Azure Blob | `CentaurTestSuite ` | `azureBlobTestCases` | -| WDL Upgrade | `WdlUpgradeTestCaseSpec` | `standardTestCases`**** | -| (other) | `CentaurTestSuite` | `standardTestCases` | - - -\* Centaur Config always uses `centaur_application.conf` except when overridden with `papi_v2alpha1_centaur_application.conf` -or `papi_v2beta_centaur_application.conf` - ([48 preview link](https://github.com/broadinstitute/cromwell/blob/a7d0601/src/ci/bin/test.inc.sh#L455-L457)) -\*\* Cromwell Config overrides - ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L213-L221)) -\*\*\* Test Directory overrides - ([47 link](https://github.com/broadinstitute/cromwell/blob/47/src/ci/bin/test.inc.sh#L440-L449)) -\*\*\*\* Test Directory only tests tagged with `wdl_upgrade` - ([47 link](https://github.com/broadinstitute/cromwell/blob/47/centaur/src/main/resources/standardTestCases/write_lines.test#L3)) - - -- Engine Upgrade: Retrieves the [Cromwell Version](https://github.com/broadinstitute/cromwell/blob/47/project/Version.scala#L8) then retrieves the previous jar/docker-image from DockerHub. Centaur starts with the prior version, then restarts with the compiled source code. -- Horicromtal: Runs a [docker-compose](https://github.com/broadinstitute/cromwell/blob/47/src/ci/docker-compose/docker-compose-horicromtal.yml) with: - 1. db-mstr: started first - 2. sum-back: runs summarizer - 3. front-back: exposes HTTP -- Horicromtal Engine Upgrade: Combination of Horicromtal and Engine Upgrade -- PAPI Upgrade: Tests run with an older version of Papi and upon restart use a newer version of Papi -- PAPI Upgrade New Workflows: Test definition [does not run any tests](https://travis-ci.org/broadinstitute/cromwell/jobs/475378412) -- WDL Upgrade: Upgrades WDL from draft-2 to 1.0 before testing -- (other): Runs `*.test` files listing the configured backend names - -## RDBMS - -| Backend | MySQL | PostgreSQL | MariaDB | -|---------|:------:|:-----------:|:--------:| -| AWS | ✅ | | | -| Local | ✅ | ✅ | | -| PAPI V2 | ✅ | | ⭕ | -| SLURM | ✅ | | | -| TES | ✅ | | | - - -⭕ Tests Horicromtal Engine Upgrade versus standard Centaur suite - - -All backends run against MySQL. The Local backend also test PostgreSQL, allowing contributors ensure WDLs work with PostgreSQL. MariaDB is tested on a specialized upgrade, where the MySQL connector client is used first, and the MariaDB client is used after restart. diff --git a/mkdocs.yml b/mkdocs.yml index 802b6e86196..b1073c93b89 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -111,5 +111,5 @@ extra_css: extra_javascript: - js/extra.js -copyright: Copyright © 2019 Broad Institute +copyright: Copyright © 2023 Broad Institute # google_analytics: ['identifierID', 'URL.org'] From aabb89e3c724e78ea1de2ea979e35e0fdbc82ff7 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 23 Mar 2023 13:56:39 -0400 Subject: [PATCH 4/7] more tidying --- README.md | 1 - processes/external-contributions/README.MD | 18 ++---------------- processes/release_processes/README.MD | 6 +----- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index fac1541cf8e..f6e3f8e742e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -[![Build Status](https://travis-ci.com/broadinstitute/cromwell.svg?branch=develop)](https://travis-ci.com/broadinstitute/cromwell?branch=develop) [![codecov](https://codecov.io/gh/broadinstitute/cromwell/branch/develop/graph/badge.svg)](https://codecov.io/gh/broadinstitute/cromwell) ## Welcome to Cromwell diff --git a/processes/external-contributions/README.MD b/processes/external-contributions/README.MD index 3369c2ca443..0bc4a8b46a7 100644 --- a/processes/external-contributions/README.MD +++ b/processes/external-contributions/README.MD @@ -38,28 +38,14 @@ Ask the questions: ### Run CI against the PR -Note: inspired by the community answer [here](https://github.community/t5/How-to-use-Git-and-GitHub/Checkout-a-branch-from-a-fork/td-p/77). - -- Problem: our CI will only run against branches of the `broadinstitute/cromwell` repo submitted by team members. -- To turn a community contribution into a PR that travis will run against: - - Run the `scripts/make_pr_clone_branch.sh` script, providing the github PR number of the external PR as the command line argument - - Eg: `scripts/make_pr_clone_branch.sh 5333` to make a PR clone branch for PR #5333. - - Push the branch to github - - Note: The final `git push` command may not work as-is, depending on your ~/.gitconfig value of `push.default`. If it doesn't work then one of the following solutions may work: - - Setting `git`'s `push.default` config value to be '`current`'. - - Using `git push --set-upstream origin 5070_pr_clone` instead - - Using `git push origin HEAD` instead - - Create a new PR for the clone branch. Indicate that it only exists to test a community contribution. - - Example title: `[PR 5333 Clone] PR for CI only` -- If you need to re-sync the clone PR against changes on the remote branch, the same script will *update* your local reference allowing you to force-push changes to the same github branch! +- Opening a PR tiggers CI to run automatically. However, if the PR was not opened by a trusted contributor, it will wait in a Github Actions queue until a trusted contributor explicitly allows CI tests run. ### Cycle through Review and CI - If the community contributor makes changes following your reviews or the CI results: - Glance at the changes to make sure they still seem reasonable. - Make any additional comments - - Re-fetch the remote reference: `git fetch -f origin refs/pull/860/head:938_pr_clone` - - Push the changes back up to github to re-trigger the CI on your clone PR. + - Rerun CI. ### Merge the PR diff --git a/processes/release_processes/README.MD b/processes/release_processes/README.MD index 5dffe3224a5..39db03833c1 100644 --- a/processes/release_processes/README.MD +++ b/processes/release_processes/README.MD @@ -86,12 +86,8 @@ The workflow outputs its status to the console. * Check that the workflow succeeded. * Check that there's now a new Cromwell release listed [here](https://github.com/broadinstitute/cromwell/releases). -* Look [in Travis](https://app.travis-ci.com/github/broadinstitute/cromwell/branches) for the release tag build that will publish Docker images for the new version. -* Let `#dsp-batch-private` know that it's okay to resume merges to `develop`. +* Let `#dsp-workflows-private` know that it's okay to resume merges to `develop`. * Announce release in `#dsp-batch`, set expectations about when the new version will be available in Terra. -* It will take about an additional hour for the Docker image to build in Travis before its tag appears on the [Cromwell Docker Hub page](https://hub.docker.com/r/broadinstitute/cromwell/tags). - * The relevant build is the one named `XX_hotfix` in [this list](https://app.travis-ci.com/github/broadinstitute/cromwell/builds). - * See the `cromwell::build::publish_artifacts()` function for details * **One business day later,** confirm that [the Homebrew package](https://formulae.brew.sh/formula/cromwell) has the latest version. If it doesn't, start investigation by looking at [Homebrew PR's](https://github.com/Homebrew/homebrew-core/pulls?q=is%3Apr+cromwell). ### How to Deploy Cromwell in CaaS staging and CaaS prod From 46f1739f3a6c8750cddbf9f9fa6148ac8a665a2c Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Thu, 23 Mar 2023 14:40:28 -0400 Subject: [PATCH 5/7] remove zombie nav path --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index b1073c93b89..4ea08cf6fa8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -102,7 +102,6 @@ nav: - Horizontal Cromwell ("Horicromtal"): developers/bitesize/horicromtal.md - General Topics: - Simpletons: developers/bitesize/general/simpletons.md - - Continuous Integration (CI) / Continuous Delivery (CD): developers/bitesize/ci/CICD.md theme: readthedocs From d098928c3157695d41418bb6382c2fc3295bb6bc Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 24 Mar 2023 11:33:46 -0400 Subject: [PATCH 6/7] update slack channel --- processes/release_processes/README.MD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processes/release_processes/README.MD b/processes/release_processes/README.MD index 39db03833c1..146c2479620 100644 --- a/processes/release_processes/README.MD +++ b/processes/release_processes/README.MD @@ -66,7 +66,7 @@ Ensure you have at least 8GB; 4GB is not sufficient. #### Let people know the publish is underway -Post another message in `#dsp-batch-private` that the release is underway, asking everyone to hold off merges to `develop` until +Post another message in `#dsp-workflows-private` that the release is underway, asking everyone to hold off merges to `develop` until the release is published. #### Run the `publish_workflow.wdl` Workflow @@ -87,7 +87,7 @@ The workflow outputs its status to the console. * Check that the workflow succeeded. * Check that there's now a new Cromwell release listed [here](https://github.com/broadinstitute/cromwell/releases). * Let `#dsp-workflows-private` know that it's okay to resume merges to `develop`. -* Announce release in `#dsp-batch`, set expectations about when the new version will be available in Terra. +* Announce release in `#dsp-workflows`, set expectations about when the new version will be available in Terra. * **One business day later,** confirm that [the Homebrew package](https://formulae.brew.sh/formula/cromwell) has the latest version. If it doesn't, start investigation by looking at [Homebrew PR's](https://github.com/Homebrew/homebrew-core/pulls?q=is%3Apr+cromwell). ### How to Deploy Cromwell in CaaS staging and CaaS prod From 3681c8f625077e655069a50475cce060362a0d96 Mon Sep 17 00:00:00 2001 From: Tom Wiseman Date: Fri, 24 Mar 2023 13:18:43 -0400 Subject: [PATCH 7/7] Update processes/external-contributions/README.MD Co-authored-by: Miguel Covarrubias --- processes/external-contributions/README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processes/external-contributions/README.MD b/processes/external-contributions/README.MD index 0bc4a8b46a7..677ecd52235 100644 --- a/processes/external-contributions/README.MD +++ b/processes/external-contributions/README.MD @@ -38,7 +38,7 @@ Ask the questions: ### Run CI against the PR -- Opening a PR tiggers CI to run automatically. However, if the PR was not opened by a trusted contributor, it will wait in a Github Actions queue until a trusted contributor explicitly allows CI tests run. +- Opening a PR triggers CI to run automatically. However, if the PR was not opened by a trusted contributor, it will wait in a Github Actions queue until a trusted contributor explicitly allows CI tests run. ### Cycle through Review and CI