diff --git a/.circleci/config.yml b/.circleci/config.yml index f3404e7854..2d5eb1c691 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ commands: echo "Changed files:" echo "$diff_output" - if echo "$diff_output" | grep -E '<< parameters.paths >>|^[^/]+$|^.circleci/|application-services/' + if echo "$diff_output" | grep -E '<< parameters.paths >>|^[^/]+$|^.circleci/|experimenter/tests/integration/' then echo "Changes detected in << parameters.paths >> or .circleci or root directory. Running tests and linting." else @@ -121,52 +121,7 @@ jobs: command: | make schemas_check - integration_nimbus_desktop_release_targeting: - machine: - image: ubuntu-2004:2023.10.1 - docker_layer_caching: true - resource_class: large - working_directory: ~/experimenter - environment: - FIREFOX_VERSION: nimbus-firefox-release - PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 4 --reruns 1 - MOZ_REMOTE_SETTINGS_DEVTOOLS: 1 # allows us to override and set the remote settings URL - steps: - - checkout - - check_file_paths: - paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests" - - run: - name: Run integration tests - command: | - cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" - - store_artifacts: - path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - - integration_nimbus_desktop_beta_targeting: - machine: - image: ubuntu-2004:2023.10.1 - docker_layer_caching: true - resource_class: large - working_directory: ~/experimenter - environment: - FIREFOX_VERSION: nimbus-firefox-beta - PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 4 --reruns 1 - FIREFOX_BETA: 1 - MOZ_REMOTE_SETTINGS_DEVTOOLS: 1 # allows us to override and set the remote settings URL - steps: - - checkout - - check_file_paths: - paths: "experimenter/tests/firefox-desktop-beta-build.env" - - run: - name: Run integration tests - command: | - cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" - - store_artifacts: - path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - - integration_nimbus_desktop_nightly_targeting: + integration_nimbus_desktop_ui: machine: image: ubuntu-2004:2023.10.1 docker_layer_caching: true @@ -174,17 +129,16 @@ jobs: working_directory: ~/experimenter environment: FIREFOX_VERSION: nimbus-firefox-release - PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 2 --reruns 1 - UPDATE_FIREFOX_VERSION: true + PYTEST_ARGS: -m nimbus_ui -n 2 --reruns 1 steps: - checkout - check_file_paths: - paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests" + paths: "experimenter/experimenter/" - run: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -200,12 +154,12 @@ jobs: steps: - checkout - check_file_paths: - paths: "experimenter/" + paths: "experimenter/experimenter/" - run: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -221,12 +175,12 @@ jobs: steps: - checkout - check_file_paths: - paths: "experimenter/" + paths: "experimenter/experimenter/" - run: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm @@ -243,17 +197,17 @@ jobs: steps: - checkout - check_file_paths: - paths: "experimenter/" + paths: "experimenter/experimenter/|experimenter/tests/firefox_desktop_release_build.env|experimenter/tests/firefox_desktop_beta_build.env" - run: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" no_output_timeout: 30m - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - integration_nimbus_desktop_ui: + integration_nimbus_desktop_release_targeting: machine: image: ubuntu-2004:2023.10.1 docker_layer_caching: true @@ -261,60 +215,44 @@ jobs: working_directory: ~/experimenter environment: FIREFOX_VERSION: nimbus-firefox-release - PYTEST_ARGS: -m nimbus_ui -n 2 --reruns 1 + PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 4 --reruns 1 + MOZ_REMOTE_SETTINGS_DEVTOOLS: 1 # allows us to override and set the remote settings URL steps: - checkout - check_file_paths: - paths: "experimenter/" + paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests/firefox_desktop_release_build.env|experimenter/tests/firefox_desktop_beta_build.env" - run: name: Run integration tests command: | cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" - - store_artifacts: - path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - - integration_nimbus_sdk_targeting: - machine: - image: ubuntu-2004:2023.10.1 - docker_layer_caching: true - resource_class: medium - working_directory: ~/experimenter - steps: - - checkout - - check_file_paths: - paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests" - - run: - name: Run rust integration tests - command: | - cp .env.integration-tests .env - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_rust + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - integration_nimbus_cirrus: + integration_nimbus_desktop_beta_targeting: machine: image: ubuntu-2004:2023.10.1 docker_layer_caching: true resource_class: large working_directory: ~/experimenter environment: - FIREFOX_VERSION: nimbus-firefox-release - PYTEST_ARGS: -k DEMO_APP -m cirrus_enrollment --reruns 1 + FIREFOX_VERSION: nimbus-firefox-beta + PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 4 --reruns 1 + FIREFOX_BETA: 1 + MOZ_REMOTE_SETTINGS_DEVTOOLS: 1 # allows us to override and set the remote settings URL steps: - checkout - check_file_paths: - paths: "experimenter/" + paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests/firefox_desktop_release_build.env|experimenter/tests/firefox_desktop_beta_build.env" - run: name: Run integration tests command: | cp .env.integration-tests .env - export CIRRUS=1 - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="$PYTEST_ARGS" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - integration_legacy: + integration_nimbus_desktop_nightly_targeting: machine: image: ubuntu-2004:2023.10.1 docker_layer_caching: true @@ -322,19 +260,21 @@ jobs: working_directory: ~/experimenter environment: FIREFOX_VERSION: nimbus-firefox-release + PYTEST_ARGS: -k FIREFOX_DESKTOP -m run_targeting -n 2 --reruns 1 + UPDATE_FIREFOX_VERSION: true steps: - checkout - check_file_paths: - paths: "experimenter/" + paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests/firefox_desktop_release_build.env|experimenter/tests/firefox_desktop_beta_build.env" - run: name: Run integration tests command: | - cp .env.sample .env - make refresh up_prod_detached integration_test_legacy + cp .env.integration-tests .env + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" - store_artifacts: path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm - integration_test_android_fenix: + integration_nimbus_fenix_enrollment: executor: name: android/android-machine resource-class: xlarge @@ -344,7 +284,7 @@ jobs: steps: - checkout - check_file_paths: - paths: "experimenter/tests/firefox-fenix-build.env" + paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|experimenter/tests/firefox_fenix_beta_build.env|experimenter/tests/firefox_fenix_release_build.env" - attach_workspace: at: /tmp/experimenter - android/accept-licenses @@ -371,7 +311,7 @@ jobs: - store_artifacts: path: /home/circleci/project/test-reports/report.htm - integration_test_ios_fennec: + integration_nimbus_ios_enrollment: macos: xcode: 15.4.0 parameters: @@ -380,7 +320,7 @@ jobs: steps: - checkout - check_file_paths: - paths: << parameters.file_path >> + paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|<< parameters.file_path >>" - macos/preboot-simulator: version: "17.5" platform: "iOS" @@ -415,7 +355,67 @@ jobs: - store_artifacts: path: ~/project/firefox-ios/firefox-ios/firefox-ios-tests/Tests/ExperimentIntegrationTests/results/index.html - create_fenix_fennec_recipes: + integration_nimbus_sdk_targeting: + machine: + image: ubuntu-2004:2023.10.1 + docker_layer_caching: true + resource_class: medium + working_directory: ~/experimenter + steps: + - checkout + - check_file_paths: + paths: "experimenter/experimenter/targeting|experimenter/experimenter/experiments|application-services/" + - run: + name: Run rust integration tests + command: | + cp .env.integration-tests .env + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_sdk + - store_artifacts: + path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm + + integration_nimbus_cirrus: + machine: + image: ubuntu-2004:2023.10.1 + docker_layer_caching: true + resource_class: large + working_directory: ~/experimenter + environment: + FIREFOX_VERSION: nimbus-firefox-release + PYTEST_ARGS: -k DEMO_APP -m cirrus_enrollment --reruns 1 + steps: + - checkout + - check_file_paths: + paths: "experimenter/experimenter/experiments/|cirrus/|application-services/" + - run: + name: Run integration tests + command: | + cp .env.integration-tests .env + export CIRRUS=1 + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="$PYTEST_ARGS" + - store_artifacts: + path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm + + integration_legacy: + machine: + image: ubuntu-2004:2023.10.1 + docker_layer_caching: true + resource_class: large + working_directory: ~/experimenter + environment: + FIREFOX_VERSION: nimbus-firefox-release + steps: + - checkout + - check_file_paths: + paths: "experimenter/" + - run: + name: Run integration tests + command: | + cp .env.sample .env + make refresh up_prod_detached integration_test_legacy + - store_artifacts: + path: ~/experimenter/experimenter/tests/integration/test-reports/report.htm + + create_mobile_recipes: working_directory: ~/experimenter machine: image: ubuntu-2004:2023.10.1 @@ -427,7 +427,7 @@ jobs: command: | cp .env.integration-tests .env export CIRRUS=1 - make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus PYTEST_ARGS="-k test_create_mobile_experiment_for_integration_test" + make refresh SKIP_DUMMY=1 PYTEST_SENTRY_DSN=$PYTEST_SENTRY_DSN CIRCLECI=$CIRCLECI up_prod_detached integration_test_nimbus_desktop PYTEST_ARGS="-k test_create_mobile_experiment_for_integration_test" - persist_to_workspace: root: experimenter paths: @@ -464,7 +464,6 @@ jobs: docker tag experimenter:deploy ${DOCKER_IMAGE}:sha-${GIT_SHA} docker push ${DOCKER_IMAGE}:sha-${GIT_SHA} - deploy_cirrus: working_directory: ~/cirrus machine: @@ -598,7 +597,7 @@ jobs: echo "No config changes, skipping" fi - check_external_firefox_integrations: + update_firefox_versions: machine: image: ubuntu-2004:2023.10.1 docker_layer_caching: true @@ -696,7 +695,7 @@ workflows: - update_external_configs - update_application_services - check_firefox_integrations: + update_firefox: triggers: - schedule: cron: "0 12 * * *" @@ -705,7 +704,7 @@ workflows: only: - main jobs: - - check_external_firefox_integrations + - update_firefox_versions build: jobs: @@ -723,30 +722,12 @@ workflows: branches: ignore: - main - - create_fenix_fennec_recipes: - name: Create fenix and fennec recipes + - create_mobile_recipes: + name: Create Fenix and iOS recipes filters: branches: only: - - check_external_firefox_integrations - - integration_nimbus_desktop_release_targeting: - name: Test Desktop Targeting (Release Firefox) - filters: - branches: - ignore: - - main - - integration_nimbus_desktop_beta_targeting: - name: Test Desktop Targeting (Beta Firefox) - filters: - branches: - ignore: - - main - - integration_nimbus_desktop_nightly_targeting: - name: Test Desktop Targeting (Nightly Firefox) - filters: - branches: - ignore: - - main + - update_firefox_versions - integration_nimbus_desktop_ui: name: Test Desktop Nimbus UI (Release Firefox) filters: @@ -765,57 +746,81 @@ workflows: branches: ignore: - main - - integration_nimbus_desktop_enrollment: - name: Test Desktop Enrollment (Release Firefox) + - integration_nimbus_desktop_release_targeting: + name: Test Desktop Targeting (Release Firefox) filters: branches: ignore: - main - - integration_nimbus_sdk_targeting: - name: Test SDK Targeting (Release Firefox) + - integration_nimbus_desktop_beta_targeting: + name: Test Desktop Targeting (Beta Firefox) filters: branches: ignore: - main - - integration_nimbus_cirrus: - name: Test Demo app with Cirrus + - integration_nimbus_desktop_nightly_targeting: + name: Test Desktop Targeting (Nightly Firefox) filters: branches: ignore: - main - - integration_legacy: - name: Test Legacy Desktop (Release Firefox) + - integration_nimbus_desktop_enrollment: + name: Test Desktop Enrollment (Release Firefox) filters: branches: ignore: - main - - integration_test_android_fenix: + - build_firefox_fenix: + name: Build Fenix APKs + filters: + branches: + only: + - update_firefox_versions + - integration_nimbus_fenix_enrollment: name: Test Firefox for Android (Fenix) requires: - - Create fenix and fennec recipes + - Create Fenix and iOS recipes - Build Fenix APKs filters: branches: only: - - check_external_firefox_integrations - - integration_test_ios_fennec: - name: Test Firefox for iOS (Fennec) Beta + - update_firefox_versions + - integration_nimbus_ios_enrollment: + name: Test Firefox for iOS Beta requires: - - Create fenix and fennec recipes + - Create Fenix and iOS recipes file_path: experimenter/tests/firefox_fennec_beta_build.env filters: branches: only: - - check_external_firefox_integrations - - integration_test_ios_fennec: - name: Test Firefox for iOS (Fennec) Release + - update_firefox_versions + - integration_nimbus_ios_enrollment: + name: Test Firefox for iOS Release requires: - - Create fenix and fennec recipes + - Create Fenix and iOS recipes file_path: experimenter/tests/firefox_fennec_release_build.env filters: branches: only: - - check_external_firefox_integrations + - update_firefox_versions + - integration_nimbus_sdk_targeting: + name: Test SDK Targeting (Release Firefox) + filters: + branches: + ignore: + - main + - integration_nimbus_cirrus: + name: Test Demo app with Cirrus + filters: + branches: + ignore: + - main + - integration_legacy: + name: Test Legacy Desktop (Release Firefox) + filters: + branches: + ignore: + - main - deploy_experimenter: name: Deploy Experimenter context: @@ -839,9 +844,3 @@ workflows: filters: branches: only: main - - build_firefox_fenix: - name: Build Fenix APKs - filters: - branches: - only: - - check_external_firefox_integrations diff --git a/Makefile b/Makefile index bf81c52a12..739f20ff68 100644 --- a/Makefile +++ b/Makefile @@ -230,10 +230,10 @@ integration_vnc_up_detached: build_prod integration_test_legacy: build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION_RUN) firefox sh -c "./experimenter/tests/experimenter_legacy_tests.sh" -integration_test_nimbus: build_prod +integration_test_nimbus_desktop: build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION_RUN) firefox sh -c "UPDATE_FIREFOX_VERSION=$(UPDATE_FIREFOX_VERSION) FIREFOX_BETA=$(FIREFOX_BETA) FIREFOX_RELEASE=$(FIREFOX_RELEASE) PYTEST_SENTRY_DSN=$(PYTEST_SENTRY_DSN) PYTEST_SENTRY_ALWAYS_REPORT=$(PYTEST_SENTRY_ALWAYS_REPORT) CIRCLECI=$(CIRCLECI) ./experimenter/tests/nimbus_integration_tests.sh" -integration_test_nimbus_rust: build_integration_test build_prod +integration_test_nimbus_sdk: build_integration_test build_prod MOZ_HEADLESS=1 $(COMPOSE_INTEGRATION_RUN) -it rust-sdk sh -c "./experimenter/tests/nimbus_rust_tests.sh" integration_test_nimbus_fenix: diff --git a/README.md b/README.md index e432b95a34..27bf014a1b 100644 --- a/README.md +++ b/README.md @@ -434,13 +434,13 @@ For a full reference of all the common commands that can be run inside the conta Run the integration test suite for experimenter inside a containerized instance of Firefox. You must also be already running a `make up` dev instance in another shell to run the integration tests. -#### make FIREFOX_VERSION integration_test_nimbus +#### make FIREFOX_VERSION integration_test_nimbus_desktop Run the integration test suite for nimbus inside a containerized instance of Firefox. You must also be already running a `make up` dev instance in another shell to run the integration tests. FIREFOX_VERSION should either be `nimbus-firefox-release` or `nimbus-firefox-beta`. If you want to run your tests against nightly, please set the variable `UPDATE_FIREFOX_VERSION` to `true` and include it in the make command. -#### make FIREFOX_VERSION integration_test_nimbus_rust +#### make FIREFOX_VERSION integration_test_nimbus_sdk Run the Nimbus SDK integration tests, which tests the advanced targeting configurations against the Nimbus SDK. @@ -529,7 +529,7 @@ FIREFOX_VERSION=nimbus-firefox-release An example for above: ```sh -make FIREFOX_VERSION=nimbus-firefox-release integration_test_nimbus PYTEST_ARGS=ktest_rollout_create_and_update +make FIREFOX_VERSION=nimbus-firefox-release integration_test_nimbus_desktop PYTEST_ARGS=ktest_rollout_create_and_update ``` #### make integration_sdk_shell diff --git a/scripts/external_integration_updater_script.sh b/scripts/external_integration_updater_script.sh index f055fe1d53..b642f95884 100755 --- a/scripts/external_integration_updater_script.sh +++ b/scripts/external_integration_updater_script.sh @@ -8,7 +8,7 @@ CURLFLAGS=("--proto" "=https" "--tlsv1.2" "-sS") git checkout main git checkout main git pull origin main -git checkout -B check_external_firefox_integrations +git checkout -B update_firefox_versions firefox_types=("fenix_beta" "fenix_nightly" "fennec_release" "fennec_beta" "desktop_beta" "desktop_release") fetch_task_info() { @@ -110,9 +110,9 @@ done if (($(git status --porcelain | wc -c) > 0)); then git add . - git commit -m "chore(nimbus): Check external firefox integrations and task ids and versions" - git push origin -f check_external_firefox_integrations - gh pr create -t "chore(nimbus): Check external firefox integrations and task ids and versions" -b "" --base main --head check_external_firefox_integrations --repo mozilla/experimenter || echo "PR already exists, skipping" + git commit -m "chore(nimbus): Update Firefox Versions" + git push origin -f update_firefox_versions + gh pr create -t "chore(nimbus): Update Firefox Versions" -b "" --base main --head update_firefox_versions --repo mozilla/experimenter || echo "PR already exists, skipping" else echo "No config changes, skipping" fi