From a91dbf92d8a9e7c2c2544cc568745ada278e9a87 Mon Sep 17 00:00:00 2001 From: Benjamin Forehand Jr Date: Tue, 9 Apr 2024 20:46:46 -0500 Subject: [PATCH 1/4] chore(nimbus): Fix for macos integration-tests. --- .circleci/config.yml | 24 ++++++++++++------------ Makefile | 6 +++++- README.md | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dceafb2e7e..cace6fd507 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,7 +137,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -159,7 +159,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -181,7 +181,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -202,7 +202,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -223,7 +223,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -244,7 +244,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -265,7 +265,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -286,7 +286,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -308,7 +308,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" no_output_timeout: 30m - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -330,7 +330,7 @@ jobs: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -350,7 +350,7 @@ jobs: name: Run rust integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_rust PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci_rust PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -372,7 +372,7 @@ jobs: command: | cp .env.integration-tests .env export CIRRUS=1 - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm diff --git a/Makefile b/Makefile index 33694521f7..2b9fbf2b76 100644 --- a/Makefile +++ b/Makefile @@ -220,9 +220,13 @@ integration_vnc_up_detached: build_prod integration_test_legacy: build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION) run firefox sh -c "sudo apt-get -qqy update && sudo apt-get -qqy install tox;sudo chmod -R a+rwx /code/experimenter/tests/integration/;sudo mkdir -m a+rwx /code/experimenter/tests/integration/test-reports;tox -c experimenter/tests/integration -e integration-test-legacy $(TOX_ARGS) -- -n 2 $(PYTEST_ARGS)" -integration_test_nimbus: build_prod +integration_test_nimbus_ci: build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION) run firefox sh -c "if [ \"$$UPDATE_FIREFOX_VERSION\" = \"true\" ]; then sudo ./experimenter/tests/integration/nimbus/utils/nightly-install.sh; fi; firefox -V; sudo apt-get -qqy update && sudo apt-get -qqy install tox;sudo chmod a+rwx /code/experimenter/tests/integration/.tox;sudo mkdir -m a+rwx /code/experimenter/tests/integration/test-reports;PYTEST_SENTRY_DSN=$(PYTEST_SENTRY_DSN) PYTEST_SENTRY_ALWAYS_REPORT=$(PYTEST_SENTRY_ALWAYS_REPORT) CIRCLECI=$(CIRCLECI) tox -c experimenter/tests/integration -e integration-test-nimbus $(TOX_ARGS) -- $(PYTEST_ARGS)" +integration_test_nimbus_dev: + $(COMPOSE_INTEGRATION) up -d + $(COMPOSE_INTEGRATION) exec firefox sh -c "if [ \"$$UPDATE_FIREFOX_VERSION\" = \"true\" ]; then sudo ./experimenter/tests/integration/nimbus/utils/nightly-install.sh; fi; firefox -V; sudo apt-get -qqy update && sudo apt-get -qqy install tox;sudo chmod a+rwx /code/experimenter/tests/integration/.tox;sudo mkdir -m a+rwx /code/experimenter/tests/integration/test-reports;PYTEST_SENTRY_DSN=$(PYTEST_SENTRY_DSN) PYTEST_SENTRY_ALWAYS_REPORT=$(PYTEST_SENTRY_ALWAYS_REPORT) CIRCLECI=$(CIRCLECI) tox -c experimenter/tests/integration -e integration-test-nimbus $(TOX_ARGS) -- $(PYTEST_ARGS)" + integration_test_nimbus_rust: build_integration_test build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION) run -it rust-sdk tox -vv -c experimenter/tests/integration -e integration-test-nimbus-rust $(TOX_ARGS) -- -n 2 $(PYTEST_ARGS) diff --git a/README.md b/README.md index 53c3ea33f1..8a4c7b5d8a 100644 --- a/README.md +++ b/README.md @@ -427,16 +427,32 @@ yarn workspace @experimenter/nimbus-ui test:cov For a full reference of all the common commands that can be run inside the container, refer to [this section of the Makefile](https://github.com/mozilla/experimenter/blob/main/Makefile#L16-L38) +### Testing + #### make integration_test_legacy Run the integration test suite for experimenter inside a containerized instance of Firefox. You must also be already running a `make up` dev instance in another shell to run the integration tests. -#### make FIREFOX_VERSION integration_test_nimbus +#### make FIREFOX_VERSION integration_test_nimbus_dev Run the integration test suite for nimbus inside a containerized instance of Firefox. You must also be already running a `make up` dev instance in another shell to run the integration tests. FIREFOX_VERSION should either be `nimbus-firefox-release` or `nimbus-firefox-beta`. If you want to run your tests against nightly, please set the variable `UPDATE_FIREFOX_VERSION` to `true` and include it in the make command. +You must run the following commands before this: + +```bash +make refresh build_integration_tests up_prod_detached +``` + +After this you can view the test run at `http://localhost:7902`, the password is `secret`. You can now make changes and run your test directly by changing the `PYTEST_ARGS` variable. +Example: +```bash +make integration_test_nimbus_dev PYTEST_ARGS="-k test-name-here" +``` + +Be sure to add the client you want to test against, refer to the tests in the `.circleci/config.yml` file for examples of clients. Put your test name followed by the client in square brackets, `PYTEST_ARGS"-k test-name-here[FIREFOX_DESKTOP]"` + #### make FIREFOX_VERSION integration_test_nimbus_rust Run the Nimbus SDK integration tests, which tests the advanced targeting configurations against the Nimbus SDK. From 840af5e4a44cee24990379f46a2230fcf6ace512 Mon Sep 17 00:00:00 2001 From: Benjamin Forehand Jr Date: Tue, 9 Apr 2024 20:52:47 -0500 Subject: [PATCH 2/4] Fix rust command. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cace6fd507..306c0a4e3d 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -350,7 +350,7 @@ jobs: name: Run rust integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_ci_rust PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_rust PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm From c2398474f62b8dff681cbf7315782bb8d4db64cf Mon Sep 17 00:00:00 2001 From: Benjamin Forehand Jr Date: Mon, 6 May 2024 09:27:37 -0500 Subject: [PATCH 3/4] Add updates to Makefile. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2b9fbf2b76..9a7e162854 100644 --- a/Makefile +++ b/Makefile @@ -225,7 +225,7 @@ integration_test_nimbus_ci: build_prod integration_test_nimbus_dev: $(COMPOSE_INTEGRATION) up -d - $(COMPOSE_INTEGRATION) exec firefox sh -c "if [ \"$$UPDATE_FIREFOX_VERSION\" = \"true\" ]; then sudo ./experimenter/tests/integration/nimbus/utils/nightly-install.sh; fi; firefox -V; sudo apt-get -qqy update && sudo apt-get -qqy install tox;sudo chmod a+rwx /code/experimenter/tests/integration/.tox;sudo mkdir -m a+rwx /code/experimenter/tests/integration/test-reports;PYTEST_SENTRY_DSN=$(PYTEST_SENTRY_DSN) PYTEST_SENTRY_ALWAYS_REPORT=$(PYTEST_SENTRY_ALWAYS_REPORT) CIRCLECI=$(CIRCLECI) tox -c experimenter/tests/integration -e integration-test-nimbus $(TOX_ARGS) -- $(PYTEST_ARGS)" + INTEGRATION_TEST_NGINX_URL="https://localhost" INTEGRATION_TEST_KINTO_URL="http://localhost:8888/v1" tox -c experimenter/tests/integration -e integration-test-nimbus-local $(TOX_ARGS) -- $(PYTEST_ARGS) integration_test_nimbus_rust: build_integration_test build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION) run -it rust-sdk tox -vv -c experimenter/tests/integration -e integration-test-nimbus-rust $(TOX_ARGS) -- -n 2 $(PYTEST_ARGS) From ce38577e8b1ea6d83a790ba5c56222df4ad71069 Mon Sep 17 00:00:00 2001 From: Benjamin Forehand Jr Date: Mon, 6 May 2024 09:33:22 -0500 Subject: [PATCH 4/4] Update readme to remove VNC steps. --- README.md | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 8a4c7b5d8a..7fcbfdcd35 100644 --- a/README.md +++ b/README.md @@ -459,34 +459,6 @@ Run the Nimbus SDK integration tests, which tests the advanced targeting configu FIREFOX_VERSION should either be `nimbus-firefox-release` or `nimbus-firefox-beta`. If you want to run your tests against nightly, please set the variable `UPDATE_FIREFOX_VERSION` to `true` and include it in the make command. -#### make FIREFOX_VERSION integration_vnc_up - -First start a prod instance of Experimenter with: - -```bash -make refresh&&make up_prod_detached -``` - -Then start the VNC service: - -```bash -make FIREFOX_VERSION integration_vnc_up -``` - -Then open your VNC client (Safari does this on OSX or just use [VNC Viewer](https://www.realvnc.com/en/connect/download/viewer/)) and open `vnc://localhost:5900` with password `secret`. Right click on the desktop and select `Applications > Shell > Bash` and enter: - -```bash -cd experimenter -sudo apt get update -sudo apt install tox -chmod a+rwx tests/integration/.tox -tox -c tests/integration/ -e integration-test-nimbus -``` - -This should run the integration tests and watch them run in a Firefox instance you can watch and interact with. - -To use NoVNC, navgate to this url `http://localhost:7902` with the password `secret`. Then you can follow the same steps as above. - ### Running Integration tests locally 1. Install [geckodriver](https://github.com/mozilla/geckodriver/releases) and have it available in your path. You should be able to run `geckodriver --version` from your command line. On MacOS you can do `brew install geckodriver` if you have homebrew installed. @@ -522,6 +494,11 @@ Firefox should pop up and start running through your test! You can change the fi firefox_options.binary = "path/to/firefox-bin" ``` +You can also do this using a make command, specifically `integration_test_nimbus_dev`. The following is an example that will run the `remote_settings` tests. +```sh +PYTEST_ARGS="-k FIREFOX_DESKTOP -m remote_settings --reruns 1" make integration_test_nimbus_dev +``` + #### Integration Test options - `TOX_ARGS`: [Tox](https://tox.readthedocs.io/en/latest/config.html#tox) commandline variables.