From dab509fceea21d7072e2e626b9c2bf7315038b2f Mon Sep 17 00:00:00 2001 From: Alexandros Koumparoulis <153118171+akoumpa@users.noreply.github.com> Date: Mon, 21 Oct 2024 01:18:12 -0700 Subject: [PATCH] disable dynamo for ddp checker (#10961) Signed-off-by: Alexandros Koumparoulis --- .github/workflows/cicd-main.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 4d201b9c55a8..345482e9a1a8 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -3887,7 +3887,7 @@ jobs: rm -rf tests/collections/llm/gpt_pretrain_results rm -rf tests/collections/llm/gpt_index_mappings - OPTIONAL_L2_NeMo_2_GPT_DDP_Param_Parity_check: + L2_NeMo_2_GPT_DDP_Param_Parity_check: needs: [cicd-test-container-setup] uses: ./.github/workflows/_test_template.yml if: contains(fromJSON(needs.cicd-test-container-setup.outputs.test_to_run), 'OPTIONAL_L2_NeMo_2_GPT_DDP_Param_Parity_check') || needs.cicd-test-container-setup.outputs.all == 'true' @@ -3895,7 +3895,7 @@ jobs: RUNNER: self-hosted-azure SCRIPT: | - python tests/lightning/test_ddp_parity_checker.py \ + TORCHDYNAMO_DISABLE=1 python tests/lightning/test_ddp_parity_checker.py \ --vocab-path=/home/TestData/nlp/megatron_gpt/data/gpt/vocab.json \ --merges-path=/home/TestData/nlp/megatron_gpt/data/gpt/merges.txt \ --data-path=/home/TestData/nlp/megatron_gpt/data/gpt/simple_wiki_gpt_preproc_text_document @@ -3903,8 +3903,7 @@ jobs: AFTER_SCRIPT: | rm -rf tests/collections/llm/gpt_pretrain_results rm -rf tests/collections/llm/gpt_index_mappings - IS_OPTIONAL: true - + L2_NeMo_2_SSM_Pretraining: needs: [cicd-test-container-setup] uses: ./.github/workflows/_test_template.yml @@ -4423,7 +4422,7 @@ jobs: - Speech_Checkpoints_tests - L2_Stable_Diffusion_Training - L2_NeMo_2_GPT_Pretraining_no_transformer_engine - #- OPTIONAL_L2_NeMo_2_GPT_DDP_Param_Parity_check + - L2_NeMo_2_GPT_DDP_Param_Parity_check - L2_NeMo_2_HF_MODEL_IMPORT - L2_NeMo_2_SSM_Pretraining - L2_NeMo_2_SSM_Finetuning @@ -4587,4 +4586,4 @@ jobs: - name: "Pipeline not successful, set exit code to 1" if: ${{ always() && steps.pipeline-conclusion.outputs.SUCCESS == 'false' }} - run: exit 1 \ No newline at end of file + run: exit 1