From c5694fd203952d0abd3bf783c49659ff23da2009 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Mon, 28 Nov 2022 18:42:43 +0100 Subject: [PATCH 01/25] Durations, disable warnings and exit -1 --- .github/actions/azureml-test/action.yml | 3 ++ .../ci/azureml_tests/run_groupwise_pytest.py | 24 ++++++--- .../submit_groupwise_azureml_pytest.py | 54 ++++++++++++------- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index c441fbf510..47018e4f15 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -98,6 +98,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --pytestargs "--durations 0 --disable-warnings" || exit -1 - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') @@ -107,6 +108,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --pytestargs "--durations 0 --disable-warnings" || exit -1 - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') @@ -116,6 +118,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --pytestargs "--durations 0 --disable-warnings" || exit -1 - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} diff --git a/tests/ci/azureml_tests/run_groupwise_pytest.py b/tests/ci/azureml_tests/run_groupwise_pytest.py index 96472b7d52..6cd5791c57 100644 --- a/tests/ci/azureml_tests/run_groupwise_pytest.py +++ b/tests/ci/azureml_tests/run_groupwise_pytest.py @@ -7,14 +7,12 @@ are set otherwise. """ -import logging -import os import sys -from azureml.core import Run +import logging import pytest -import json import argparse import glob +from azureml.core import Run from test_groups import nightly_test_groups, unit_test_groups if __name__ == "__main__": @@ -37,6 +35,13 @@ default="group_cpu_001", help="Group name for the tests", ) + parser.add_argument( + "--pytestargs", + "-a", + nargs="+", + default="", + help="Additional arguments to pass to pytest as a string. Multiple arguments should be separated by space.", + ) args = parser.parse_args() if args.testkind == "nightly": @@ -56,8 +61,8 @@ logger.info("Executing tests now...") # execute pytest command - pytest_exit_code = pytest.main(test_group) - + pytest_exit_code = pytest.main(args=[test_group, args.pytestargs]) + logger.info("Test execution completed!") # log pytest exit code as a metric @@ -70,5 +75,8 @@ # logger.info("os.listdir files {}".format(os.listdir("."))) # upload pytest stdout file - logs_path = (glob.glob('**/70_driver_log.txt', recursive=True) + glob.glob('**/std_log.txt', recursive=True))[0] - run.upload_file(name='test_logs', path_or_stream=logs_path) + logs_path = ( + glob.glob("**/70_driver_log.txt", recursive=True) + + glob.glob("**/std_log.txt", recursive=True) + )[0] + run.upload_file(name="test_logs", path_or_stream=logs_path) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 61a688bd02..74d645d3ed 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -25,6 +25,7 @@ --reponame (str): the Github repository name --branch (str): the branch being run It is also possible to put any text string in these. + Example: Usually, this script is run by a DevOps pipeline. It can also be run from cmd line. @@ -96,6 +97,7 @@ def setup_workspace( # create_resource_group=True, location=location, auth=cli_auth, + show_output=False, ) return ws @@ -139,16 +141,18 @@ def setup_persistent_compute_target(workspace, cluster_name, vm_size, max_nodes) return cpu_cluster -def create_run_config(cpu_cluster, - docker_proc_type, - workspace, - add_gpu_dependencies, - add_spark_dependencies, - conda_pkg_cudatoolkit, - conda_pkg_cudnn, - conda_pkg_jdk, - conda_pkg_python, - reco_wheel_path): +def create_run_config( + cpu_cluster, + docker_proc_type, + workspace, + add_gpu_dependencies, + add_spark_dependencies, + conda_pkg_cudatoolkit, + conda_pkg_cudnn, + conda_pkg_jdk, + conda_pkg_python, + reco_wheel_path, +): """ AzureML requires the run environment to be setup prior to submission. This configures a docker persistent compute. Even though @@ -167,6 +171,7 @@ def create_run_config(cpu_cluster, added to the conda environment, else False add_spark_dependencies (bool) : True if PySpark packages should be added to the conda environment, else False + Return: run_azuremlcompute : AzureML run config """ @@ -191,7 +196,9 @@ def create_run_config(cpu_cluster, conda_dep = CondaDependencies() conda_dep.add_conda_package(conda_pkg_python) conda_dep.add_pip_package(whl_url) - conda_dep.add_pip_package("pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip") + conda_dep.add_pip_package( + "pymanopt@https://github.com/pymanopt/pymanopt/archive/fb36a272cdeecb21992cfd9271eb82baafeb316d.zip" + ) # install extra dependencies if add_gpu_dependencies and add_spark_dependencies: @@ -233,9 +240,7 @@ def create_experiment(workspace, experiment_name): return exp -def submit_experiment_to_azureml( - test, run_config, experiment, test_group, test_kind -): +def submit_experiment_to_azureml(test, run_config, experiment, test_group, test_kind): """ Submitting the experiment to AzureML actually runs the script. @@ -370,11 +375,15 @@ def create_arg_parser(): ) # flag to indicate whether gpu dependencies should be included in conda env parser.add_argument( - "--add_gpu_dependencies", action="store_true", help="include packages for GPU support" + "--add_gpu_dependencies", + action="store_true", + help="include packages for GPU support", ) - # flag to indicate whether pyspark dependencies should be included in conda env + # flag to indicate whether pyspark dependencies should be included in conda env parser.add_argument( - "--add_spark_dependencies", action="store_true", help="include packages for PySpark support" + "--add_spark_dependencies", + action="store_true", + help="include packages for PySpark support", ) # path where test logs should be downloaded parser.add_argument( @@ -417,6 +426,12 @@ def create_arg_parser(): default="unit", help="Test kind - nightly or unit", ) + parser.add_argument( + "--pytestargs", + nargs="+", + default="", + help="Additional arguments to pass to pytest as a string. Multiple arguments should be separated by space.", + ) args = parser.parse_args() return args @@ -453,7 +468,7 @@ def create_arg_parser(): max_nodes=args.maxnodes, ) - wheel_list = glob.glob('./dist/*.whl') + wheel_list = glob.glob("./dist/*.whl") if not wheel_list: logger.error("Wheel not found!") logger.info("Found wheel at " + wheel_list[0]) @@ -481,6 +496,7 @@ def create_arg_parser(): experiment=experiment, test_group=args.testgroup, test_kind=args.testkind, + pytest_args=args.pytestargs, ) # add helpful information to experiment on Azure @@ -494,6 +510,6 @@ def create_arg_parser(): # save pytest exit code metrics = run.get_metrics() with open("pytest_exit_code.log", "w") as f: - f.write(str(metrics.get('pytest_exit_code'))) + f.write(str(metrics.get("pytest_exit_code"))) run.complete() From 2d0b81393cb110b7b993e4a9c79c2f2f0d5a2f83 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Mon, 28 Nov 2022 18:48:49 +0100 Subject: [PATCH 02/25] :bug: --- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 74d645d3ed..c6aee17960 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -496,7 +496,7 @@ def create_arg_parser(): experiment=experiment, test_group=args.testgroup, test_kind=args.testkind, - pytest_args=args.pytestargs, + pytestargs=args.pytestargs, ) # add helpful information to experiment on Azure From 4d9d42d318f89aacff761b6e341ef972aa8e39b2 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Mon, 28 Nov 2022 19:04:58 +0100 Subject: [PATCH 03/25] :bug: --- .../submit_groupwise_azureml_pytest.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index c6aee17960..4e73425f77 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -137,7 +137,7 @@ def setup_persistent_compute_target(workspace, cluster_name, vm_size, max_nodes) vm_size=vm_size, max_nodes=max_nodes ) cpu_cluster = ComputeTarget.create(workspace, cluster_name, compute_config) - cpu_cluster.wait_for_completion(show_output=True) + cpu_cluster.wait_for_completion(show_output=False) return cpu_cluster @@ -240,23 +240,24 @@ def create_experiment(workspace, experiment_name): return exp -def submit_experiment_to_azureml(test, run_config, experiment, test_group, test_kind): +def submit_experiment_to_azureml( + test, run_config, experiment, test_group, test_kind, pytestargs +): """ Submitting the experiment to AzureML actually runs the script. Args: - test (str) - pytest script, folder/test - such as ./tests/ci/run_pytest.py - test_folder (str) - folder where tests to run are stored, - like ./tests/unit - test_markers (str) - test markers used by pytest - "not notebooks and not spark and not gpu" - run_config - environment configuration - experiment - instance of an Experiment, a collection of + test (str): Pytest script, folder/test such as ./tests/ci/run_pytest.py + run_config (obj): Environment configuration + experiment (obj): Instance of an Experiment, a collection of trials where each trial is a run. + test_group (str): Name of the test group. + test_kind (str): Name of the test kind, such as nightly or unit. + pytestargs (str): Pytest arguments. + Return: - run : AzureML run or trial + obj: AzureML run or trial """ project_folder = "." @@ -270,6 +271,8 @@ def submit_experiment_to_azureml(test, run_config, experiment, test_group, test_ test_group, "--testkind", test_kind, + "--pytestargs", + pytestargs, ], # docker_runtime_config=dc ) From 07c10fee399fae0b65bd4a2ab2e3a64e0f62e92d Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 29 Nov 2022 03:00:48 +0100 Subject: [PATCH 04/25] :bug: --- .github/actions/azureml-test/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 47018e4f15..7248b45d4a 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -98,7 +98,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" || exit -1 + --pytestargs "--durations 0 --disable-warnings" - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') @@ -108,7 +108,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" || exit -1 + --pytestargs "--durations 0 --disable-warnings" - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') @@ -118,7 +118,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" || exit -1 + --pytestargs "--durations 0 --disable-warnings" - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} From 6173aeeafca181240a593de059b2684e5afb27f9 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 29 Nov 2022 03:08:21 +0100 Subject: [PATCH 05/25] quiet --- .github/actions/azureml-test/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 7248b45d4a..864e178d8a 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -74,17 +74,17 @@ runs: python-version: "3.8" - name: Install azureml-core and azure-cli on a GitHub hosted server shell: bash - run: pip install azureml-core azure-cli + run: pip install --quiet azureml-core azure-cli - name: Log in to Azure uses: azure/login@v1 with: creds: ${{inputs.AZUREML_TEST_CREDENTIALS}} - name: Install wheel package shell: bash - run: pip install wheel + run: pip install --quiet wheel - name: Create wheel from setup.py shell: bash - run: python setup.py bdist_wheel + run: python setup.py --quiet bdist_wheel - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" From 23e417d5aa1218a085a744d8dedd89914722769a Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 29 Nov 2022 03:10:17 +0100 Subject: [PATCH 06/25] :bug: --- .github/actions/azureml-test/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 864e178d8a..7b4c443788 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -98,7 +98,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" + - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') @@ -108,7 +108,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" + - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') @@ -118,7 +118,6 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} From 6b0d6880eb6289d4c344f67f4299900aa5819066 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 29 Nov 2022 03:25:05 +0100 Subject: [PATCH 07/25] :bug::bug: --- .github/actions/azureml-test/action.yml | 7 ++++--- tests/ci/azureml_tests/run_groupwise_pytest.py | 2 +- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 7b4c443788..cddb60655b 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -96,9 +96,9 @@ runs: python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - + --pytestargs "--durations 0 --disable-warnings" - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') @@ -108,7 +108,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - + --pytestargs "--durations 0 --disable-warnings" - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') @@ -118,6 +118,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --pytestargs "--durations 0 --disable-warnings" - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} diff --git a/tests/ci/azureml_tests/run_groupwise_pytest.py b/tests/ci/azureml_tests/run_groupwise_pytest.py index 6cd5791c57..3e31840c98 100644 --- a/tests/ci/azureml_tests/run_groupwise_pytest.py +++ b/tests/ci/azureml_tests/run_groupwise_pytest.py @@ -38,7 +38,7 @@ parser.add_argument( "--pytestargs", "-a", - nargs="+", + action="store", default="", help="Additional arguments to pass to pytest as a string. Multiple arguments should be separated by space.", ) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 4e73425f77..1ae98b23bb 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -431,7 +431,7 @@ def create_arg_parser(): ) parser.add_argument( "--pytestargs", - nargs="+", + action="store", default="", help="Additional arguments to pass to pytest as a string. Multiple arguments should be separated by space.", ) From 6b4fcc26f47d9b0598b7d0c3dab575c192b336e0 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 12:30:33 +0100 Subject: [PATCH 08/25] updated echo instruction --- .github/actions/azureml-test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index cddb60655b..7bfb336b7d 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -87,7 +87,7 @@ runs: run: python setup.py --quiet bdist_wheel - name: Extract branch name shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: echo "Branch name: ${GITHUB_REF#refs/heads/}" id: extract_branch - name: Submit CPU tests to AzureML shell: bash @@ -125,7 +125,7 @@ runs: - name: Get exit status shell: bash id: exit_status - run: echo ::set-output name=code::$(cat ${{inputs.PYTEST_EXIT_CODE}}) + run: echo "PyTest exit code: $(cat ${{inputs.PYTEST_EXIT_CODE}})" - name: Check Success/Failure if: ${{ steps.execute_tests.outputs.code != 0 }} uses: actions/github-script@v3 From 051626139ea48da1d18b03d4b32cca51752d6aeb Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 12:32:35 +0100 Subject: [PATCH 09/25] updated echo instruction :bug: --- .github/actions/azureml-test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 7bfb336b7d..be71b3813f 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -87,7 +87,7 @@ runs: run: python setup.py --quiet bdist_wheel - name: Extract branch name shell: bash - run: echo "Branch name: ${GITHUB_REF#refs/heads/}" + run: echo "Branch name -> ${GITHUB_REF#refs/heads/}" id: extract_branch - name: Submit CPU tests to AzureML shell: bash @@ -125,7 +125,7 @@ runs: - name: Get exit status shell: bash id: exit_status - run: echo "PyTest exit code: $(cat ${{inputs.PYTEST_EXIT_CODE}})" + run: echo "PyTest exit code -> $(cat ${{inputs.PYTEST_EXIT_CODE}})" - name: Check Success/Failure if: ${{ steps.execute_tests.outputs.code != 0 }} uses: actions/github-script@v3 From a6357b3ee56c11d05f315feea627e0db0c5cd2eb Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 12:46:18 +0100 Subject: [PATCH 10/25] select right branch --- .github/actions/azureml-test/action.yml | 2 +- .github/actions/get-test-groups/action.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index be71b3813f..5882c40240 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -94,7 +94,7 @@ runs: if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index c741b98dad..127c9576e9 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -1,3 +1,8 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# --------------------------------------------------------- + name: get-test-groups description: "Get test group names from tests_groups.py" inputs: From f63bcac090c7b5dbe5d8c95904eb4f706125e760 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 12:50:45 +0100 Subject: [PATCH 11/25] select right branch --- .github/actions/azureml-test/action.yml | 4 ++-- .github/actions/get-test-groups/action.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 5882c40240..da7900cbfb 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -104,7 +104,7 @@ runs: if: contains(inputs.TEST_GROUP, 'gpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} @@ -114,7 +114,7 @@ runs: if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index 127c9576e9..b8639891f2 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -31,4 +31,4 @@ runs: else test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))') fi - echo ::set-output name=test_groups::$test_groups_str + echo "{test_groups}={$test_groups_str}" >> $GITHUB_OUTPUT From 95b8fc59f0a5f0d42b5acaa389e56c38c76b830e Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 12:53:14 +0100 Subject: [PATCH 12/25] select right branch --- .github/actions/azureml-test/action.yml | 2 +- .github/actions/get-test-groups/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index da7900cbfb..6c220c5f3a 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -114,7 +114,7 @@ runs: if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} diff --git a/.github/actions/get-test-groups/action.yml b/.github/actions/get-test-groups/action.yml index b8639891f2..127c9576e9 100644 --- a/.github/actions/get-test-groups/action.yml +++ b/.github/actions/get-test-groups/action.yml @@ -31,4 +31,4 @@ runs: else test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))') fi - echo "{test_groups}={$test_groups_str}" >> $GITHUB_OUTPUT + echo ::set-output name=test_groups::$test_groups_str From 6de3d2ea045944bb5cf8eb5a69b3ac7daf982504 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 12:56:03 +0100 Subject: [PATCH 13/25] select right branch --- .github/actions/azureml-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 6c220c5f3a..da7900cbfb 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -114,7 +114,7 @@ runs: if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} From a9b64d768ebc5b07bbdb29c238a37459a30b8b22 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 13:04:36 +0100 Subject: [PATCH 14/25] select right branch --- .github/actions/azureml-test/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index da7900cbfb..771da3afdc 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -94,7 +94,7 @@ runs: if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref_name }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} @@ -104,7 +104,7 @@ runs: if: contains(inputs.TEST_GROUP, 'gpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref_name }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} @@ -114,7 +114,7 @@ runs: if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.base_ref }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref_name }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} From e9a9a701c8413d4bbb6e01af8f3ab779e61523cd Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 13:21:59 +0100 Subject: [PATCH 15/25] select right branch --- .github/actions/azureml-test/action.yml | 46 ++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 771da3afdc..d27e6003d4 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -87,45 +87,45 @@ runs: run: python setup.py --quiet bdist_wheel - name: Extract branch name shell: bash - run: echo "Branch name -> ${GITHUB_REF#refs/heads/}" + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: extract_branch - name: Submit CPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref_name }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} --pytestargs "--durations 0 --disable-warnings" - - name: Submit GPU tests to AzureML - shell: bash - if: contains(inputs.TEST_GROUP, 'gpu') - run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref_name }} - --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" - - name: Submit PySpark tests to AzureML - shell: bash - if: contains(inputs.TEST_GROUP, 'spark') - run: >- - python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref_name }} - --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} - --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" + # - name: Submit GPU tests to AzureML + # shell: bash + # if: contains(inputs.TEST_GROUP, 'gpu') + # run: >- + # python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} + # --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + # --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + # --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} + # --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + # --pytestargs "--durations 0 --disable-warnings" + # - name: Submit PySpark tests to AzureML + # shell: bash + # if: contains(inputs.TEST_GROUP, 'spark') + # run: >- + # python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} + # --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + # --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + # --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} + # --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + # --pytestargs "--durations 0 --disable-warnings" - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} - name: Get exit status shell: bash id: exit_status - run: echo "PyTest exit code -> $(cat ${{inputs.PYTEST_EXIT_CODE}})" + run: echo ::set-output name=code::$(cat ${{inputs.PYTEST_EXIT_CODE}}) - name: Check Success/Failure if: ${{ steps.execute_tests.outputs.code != 0 }} uses: actions/github-script@v3 From dbbb708d99fa95cdfbafc2cdab995d8546076e3b Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 13:28:03 +0100 Subject: [PATCH 16/25] focusing only on CPU tests --- .github/actions/azureml-test/action.yml | 30 +++++++++---------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index d27e6003d4..790893a16b 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -99,26 +99,16 @@ runs: --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} --pytestargs "--durations 0 --disable-warnings" - # - name: Submit GPU tests to AzureML - # shell: bash - # if: contains(inputs.TEST_GROUP, 'gpu') - # run: >- - # python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} - # --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - # --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - # --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} - # --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - # --pytestargs "--durations 0 --disable-warnings" - # - name: Submit PySpark tests to AzureML - # shell: bash - # if: contains(inputs.TEST_GROUP, 'spark') - # run: >- - # python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - # --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} - # --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} - # --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} - # --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - # --pytestargs "--durations 0 --disable-warnings" + - name: Submit GPU tests to AzureML + shell: bash + if: contains(inputs.TEST_GROUP, 'gpu') + run: >- + echo "Skipping GPU" >> ${{inputs.TEST_LOGS_PATH}} + - name: Submit PySpark tests to AzureML + shell: bash + if: contains(inputs.TEST_GROUP, 'spark') + run: >- + echo "Skipping PySpark" >> ${{inputs.TEST_LOGS_PATH}} - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} From 319dda93d814f2551ef7f958337e085cbe2e0221 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 13:38:57 +0100 Subject: [PATCH 17/25] focusing only on CPU tests --- .github/actions/azureml-test/action.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 790893a16b..dda0904cd1 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -85,16 +85,12 @@ runs: - name: Create wheel from setup.py shell: bash run: python setup.py --quiet bdist_wheel - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - name: Submit CPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'cpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} From 064e1b034e5cbcc5880205fc34f16cdedb6499fe Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 14:18:09 +0100 Subject: [PATCH 18/25] add durations and suppress warnings --- .github/actions/azureml-test/action.yml | 2 +- tests/ci/azureml_tests/run_groupwise_pytest.py | 9 +-------- .../azureml_tests/submit_groupwise_azureml_pytest.py | 12 +++++++++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index dda0904cd1..b36024acab 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -94,7 +94,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --pytestargs "--durations 0 --disable-warnings" + --durations 0 --disable-warnings - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') diff --git a/tests/ci/azureml_tests/run_groupwise_pytest.py b/tests/ci/azureml_tests/run_groupwise_pytest.py index 3e31840c98..b96899af54 100644 --- a/tests/ci/azureml_tests/run_groupwise_pytest.py +++ b/tests/ci/azureml_tests/run_groupwise_pytest.py @@ -35,13 +35,6 @@ default="group_cpu_001", help="Group name for the tests", ) - parser.add_argument( - "--pytestargs", - "-a", - action="store", - default="", - help="Additional arguments to pass to pytest as a string. Multiple arguments should be separated by space.", - ) args = parser.parse_args() if args.testkind == "nightly": @@ -61,7 +54,7 @@ logger.info("Executing tests now...") # execute pytest command - pytest_exit_code = pytest.main(args=[test_group, args.pytestargs]) + pytest_exit_code = pytest.main(args=test_group) logger.info("Test execution completed!") diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 1ae98b23bb..1cc4f74f6d 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -429,11 +429,17 @@ def create_arg_parser(): default="unit", help="Test kind - nightly or unit", ) + # Flag to indicate whether to turn off the warnings parser.add_argument( - "--pytestargs", + "--disable-warnings", + action="store_true", + help="Turn off warnings", + ) + parser.add_argument( + "--durations", action="store", - default="", - help="Additional arguments to pass to pytest as a string. Multiple arguments should be separated by space.", + default=0, + help="Add the duration of the test to the output", ) args = parser.parse_args() From f0d1b44e1fed43537ff2e1f2d4556c8408882d5c Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Wed, 1 Feb 2023 14:22:05 +0100 Subject: [PATCH 19/25] add durations and suppress warnings :bug: --- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 1cc4f74f6d..6be441207b 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -240,9 +240,7 @@ def create_experiment(workspace, experiment_name): return exp -def submit_experiment_to_azureml( - test, run_config, experiment, test_group, test_kind, pytestargs -): +def submit_experiment_to_azureml(test, run_config, experiment, test_group, test_kind): """ Submitting the experiment to AzureML actually runs the script. @@ -271,8 +269,6 @@ def submit_experiment_to_azureml( test_group, "--testkind", test_kind, - "--pytestargs", - pytestargs, ], # docker_runtime_config=dc ) @@ -505,7 +501,6 @@ def create_arg_parser(): experiment=experiment, test_group=args.testgroup, test_kind=args.testkind, - pytestargs=args.pytestargs, ) # add helpful information to experiment on Azure From 5c543f60d6b7d92c63868a9a1df847135277559f Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 9 Feb 2023 20:56:46 +0100 Subject: [PATCH 20/25] adding durations --- .github/actions/azureml-test/action.yml | 2 +- tests/ci/azureml_tests/run_groupwise_pytest.py | 5 +++-- tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py | 6 ------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index b36024acab..c257322359 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -94,7 +94,7 @@ runs: --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} - --durations 0 --disable-warnings + --disable-warnings - name: Submit GPU tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'gpu') diff --git a/tests/ci/azureml_tests/run_groupwise_pytest.py b/tests/ci/azureml_tests/run_groupwise_pytest.py index b96899af54..2223f764cc 100644 --- a/tests/ci/azureml_tests/run_groupwise_pytest.py +++ b/tests/ci/azureml_tests/run_groupwise_pytest.py @@ -53,8 +53,9 @@ logger.info(str(sys.version)) logger.info("Executing tests now...") - # execute pytest command - pytest_exit_code = pytest.main(args=test_group) + # Execute pytest command + pytest_string = test_group + ["--durations"] + ["0"] + pytest_exit_code = pytest.main(pytest_string) logger.info("Test execution completed!") diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index 6be441207b..e8bc2c7518 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -431,12 +431,6 @@ def create_arg_parser(): action="store_true", help="Turn off warnings", ) - parser.add_argument( - "--durations", - action="store", - default=0, - help="Add the duration of the test to the output", - ) args = parser.parse_args() return args From 315082402f89b67b41740963e54089579ec06821 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Fri, 10 Feb 2023 07:11:34 +0100 Subject: [PATCH 21/25] adding warnings --- tests/ci/azureml_tests/run_groupwise_pytest.py | 12 +++++++++++- .../submit_groupwise_azureml_pytest.py | 18 +++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/ci/azureml_tests/run_groupwise_pytest.py b/tests/ci/azureml_tests/run_groupwise_pytest.py index 2223f764cc..719c9022ce 100644 --- a/tests/ci/azureml_tests/run_groupwise_pytest.py +++ b/tests/ci/azureml_tests/run_groupwise_pytest.py @@ -35,6 +35,12 @@ default="group_cpu_001", help="Group name for the tests", ) + # Flag to indicate whether to turn off the warnings + parser.add_argument( + "--disable-warnings", + action="store_true", + help="Turn off warnings", + ) args = parser.parse_args() if args.testkind == "nightly": @@ -53,8 +59,12 @@ logger.info(str(sys.version)) logger.info("Executing tests now...") - # Execute pytest command + # Add options to pytest command (Duration and disable warnings) pytest_string = test_group + ["--durations"] + ["0"] + if args.disable_warnings is True: + pytest_string += ["--disable-warnings"] + + # Execute pytest command pytest_exit_code = pytest.main(pytest_string) logger.info("Test execution completed!") diff --git a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py index e8bc2c7518..6cdbd2da8c 100644 --- a/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py +++ b/tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py @@ -240,7 +240,9 @@ def create_experiment(workspace, experiment_name): return exp -def submit_experiment_to_azureml(test, run_config, experiment, test_group, test_kind): +def submit_experiment_to_azureml( + test, run_config, experiment, test_group, test_kind, warnings +): """ Submitting the experiment to AzureML actually runs the script. @@ -258,18 +260,15 @@ def submit_experiment_to_azureml(test, run_config, experiment, test_group, test_ obj: AzureML run or trial """ - project_folder = "." + arguments = ["--testgroup", test_group, "--testkind", test_kind] + if warnings is True: + arguments.append("--disable-warnings") script_run_config = ScriptRunConfig( - source_directory=project_folder, + source_directory=".", script=test, run_config=run_config, - arguments=[ - "--testgroup", - test_group, - "--testkind", - test_kind, - ], + arguments=arguments, # docker_runtime_config=dc ) run = experiment.submit(script_run_config) @@ -495,6 +494,7 @@ def create_arg_parser(): experiment=experiment, test_group=args.testgroup, test_kind=args.testkind, + warnings=args.disable_warnings, ) # add helpful information to experiment on Azure From a9baaa81cb675946e79f355c8b9eeea5f7a8b508 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Fri, 10 Feb 2023 07:29:53 +0100 Subject: [PATCH 22/25] add gpu and spark tests again --- .github/actions/azureml-test/action.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index c257322359..3c46982266 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -99,12 +99,22 @@ runs: shell: bash if: contains(inputs.TEST_GROUP, 'gpu') run: >- - echo "Skipping GPU" >> ${{inputs.TEST_LOGS_PATH}} + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} + --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --disable-warnings - name: Submit PySpark tests to AzureML shell: bash if: contains(inputs.TEST_GROUP, 'spark') run: >- - echo "Skipping PySpark" >> ${{inputs.TEST_LOGS_PATH}} + python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} + --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} + --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} + --disable-warnings - name: Print test logs shell: bash run: cat ${{inputs.TEST_LOGS_PATH}} From 217f63fd161d44c7f7e9c9ce9912101d9216cb46 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Fri, 10 Feb 2023 07:55:11 +0100 Subject: [PATCH 23/25] :bug: --- .github/actions/azureml-test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 3c46982266..885b0a3349 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -100,7 +100,7 @@ runs: if: contains(inputs.TEST_GROUP, 'gpu') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} @@ -110,7 +110,7 @@ runs: if: contains(inputs.TEST_GROUP, 'spark') run: >- python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}} - --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }} + --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ github.ref }} --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}} --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}} From 11349b94bd3350ecd2fcd8194f5c91a5762a3d5a Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 16 Feb 2023 20:27:12 +0100 Subject: [PATCH 24/25] restricting azureml --- .github/actions/azureml-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/azureml-test/action.yml b/.github/actions/azureml-test/action.yml index 885b0a3349..1381aae075 100644 --- a/.github/actions/azureml-test/action.yml +++ b/.github/actions/azureml-test/action.yml @@ -74,7 +74,7 @@ runs: python-version: "3.8" - name: Install azureml-core and azure-cli on a GitHub hosted server shell: bash - run: pip install --quiet azureml-core azure-cli + run: pip install --quiet "azureml-core>1,<2" "azure-cli>2,<3" - name: Log in to Azure uses: azure/login@v1 with: From ddb809775e194048e530f20079d80ea80b019515 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 16 Feb 2023 20:35:29 +0100 Subject: [PATCH 25/25] adding more concurrent jobs on GitHub actions --- .github/workflows/azureml-cpu-nightly.yml | 1 + .github/workflows/azureml-gpu-nightly.yml | 1 + .github/workflows/azureml-spark-nightly.yml | 1 + .github/workflows/azureml-unit-tests.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/azureml-cpu-nightly.yml b/.github/workflows/azureml-cpu-nightly.yml index 84f3ba7cbf..dc9b19b898 100644 --- a/.github/workflows/azureml-cpu-nightly.yml +++ b/.github/workflows/azureml-cpu-nightly.yml @@ -66,6 +66,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/.github/workflows/azureml-gpu-nightly.yml b/.github/workflows/azureml-gpu-nightly.yml index 1af9e61be6..9684a48514 100644 --- a/.github/workflows/azureml-gpu-nightly.yml +++ b/.github/workflows/azureml-gpu-nightly.yml @@ -66,6 +66,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/.github/workflows/azureml-spark-nightly.yml b/.github/workflows/azureml-spark-nightly.yml index f73a939731..6a9e50f817 100644 --- a/.github/workflows/azureml-spark-nightly.yml +++ b/.github/workflows/azureml-spark-nightly.yml @@ -66,6 +66,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }} diff --git a/.github/workflows/azureml-unit-tests.yml b/.github/workflows/azureml-unit-tests.yml index 03bde31f38..9d1c3e5074 100644 --- a/.github/workflows/azureml-unit-tests.yml +++ b/.github/workflows/azureml-unit-tests.yml @@ -53,6 +53,7 @@ jobs: name: ${{ join(matrix.*, ', ') }} runs-on: ubuntu-latest strategy: + max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration matrix: python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"'] test-group: ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}