From 761e155af50920257c5e61066c39d587675c5148 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 00:19:40 -0400 Subject: [PATCH 1/6] [wasm][tests] Add properties to allow passing args to xharness New properties: - `$(WasmXHarnessArgs)` - `$(WasmXHarnessMonoArgs)` - `$(WasmTestAppArgs)` Xharness command line is built with these: `dotnet xharness wasm test .. $(WasmXHarnessArgs) .. -- $(WasmXHarnessMonoArgs) .. --run .. $(WasmTestAppArgs)` --- .../testing/libraries/testing-wasm.md | 12 ++++++++++ eng/testing/tests.mobile.targets | 24 +++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/workflow/testing/libraries/testing-wasm.md b/docs/workflow/testing/libraries/testing-wasm.md index 3979f9033a09b4..242e6c9cee4a90 100644 --- a/docs/workflow/testing/libraries/testing-wasm.md +++ b/docs/workflow/testing/libraries/testing-wasm.md @@ -96,6 +96,18 @@ The following shows how to run tests for a specific library make -C src/mono/wasm/ run-browser-tests-System.AppContext ``` +### Passing arguments to xharness + +- `$(WasmXHarnessArgs)` - xharness command arguments + + Example: `WasmXHarnessArgs="--xyz"` -> becomes `dotnet xharness wasm test --xyz` + +- `$(WasmXHarnessMonoArgs)` - arguments to mono + + Example: `WasmXHarnessMonoArgs="--runtime-arg=--trace=E"` + +- `$(WasmTestAppArgs)` - arguments for the test app itself + ### Running outer loop tests using Browser instance To run all tests, including "outer loop" tests (which are typically slower and in some test suites less reliable, but which are more comprehensive): diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index 81246339652ca0..dbb2ced7861b7b 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -4,8 +4,6 @@ $([MSBuild]::NormalizeDirectory('$(OutDir)', 'AppBundle')) $([MSBuild]::NormalizePath('$(BundleDir)', '$(RunScriptOutputName)')) true - V8 - $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 true true @@ -13,10 +11,26 @@ true + + + V8 + $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 + + <_XHarnessArgs>wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT + + <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js + <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) + <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) %24WasmXHarnessArgs + + <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll + <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll --testing + + <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) %24WasmTestAppArgs + + $HARNESS_RUNNER $(_XHarnessArgs) -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) + + - - $HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT -- $(RunTestsJSArguments) --run WasmTestRunner.dll $(AssemblyName).dll - $HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT --expected-exit-code=$(ExpectedExitCode) -- $(RunTestsJSArguments) --run $(AssemblyName).dll --testing false true false From a7b4f0bc1af2b4b8f4a3898eabaa2743b75f7a22 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 01:07:35 -0400 Subject: [PATCH 2/6] [wasm][tests] extract wasm bits into tests.wasm.targets --- eng/testing/tests.mobile.targets | 79 +------------------------------- eng/testing/tests.wasm.targets | 78 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 77 deletions(-) create mode 100644 eng/testing/tests.wasm.targets diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index dbb2ced7861b7b..a0e795f8f7c82b 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -11,33 +11,6 @@ true - - - V8 - $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 - - <_XHarnessArgs>wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT - - <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js - <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) - <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) %24WasmXHarnessArgs - - <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll - <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll --testing - - <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) %24WasmTestAppArgs - - $HARNESS_RUNNER $(_XHarnessArgs) -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) - - - - false - true - false - false - false - - false false @@ -81,6 +54,8 @@ <_MobileIntermediateOutputPath>$(IntermediateOutputPath)mobile + + - - - - - PrepareForWasmBuildApp;$(WasmBuildAppDependsOn) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasm', 'emsdk')) - - - - - - - - - - - - - - - - - - - $(BundleDir) - WasmTestRunner.dll - $(MonoProjectRoot)\wasm\runtime-test.js - $(InvariantGlobalization) - true - false - - - - - - $([System.IO.Directory]::GetParent('%(Identity)').Name) - - - - - - - - - - - - diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets new file mode 100644 index 00000000000000..7a084957d5112e --- /dev/null +++ b/eng/testing/tests.wasm.targets @@ -0,0 +1,78 @@ + + + + V8 + $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 + + <_XHarnessArgs>wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT + + <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js + <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) + <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) %24WasmXHarnessArgs + + <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll + <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll --testing + + <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) %24WasmTestAppArgs + + $HARNESS_RUNNER $(_XHarnessArgs) -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) + + + + false + true + false + false + false + + + + + + + PrepareForWasmBuildApp;$(WasmBuildAppDependsOn) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasm', 'emsdk')) + + + + + + + + + + + + + + + + + + + $(BundleDir) + WasmTestRunner.dll + $(MonoProjectRoot)\wasm\runtime-test.js + $(InvariantGlobalization) + true + false + + + + + + $([System.IO.Directory]::GetParent('%(Identity)').Name) + + + + + + + + + + + + From e4312ef5a6821a2946b1ceabaaded2abc56781ff Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 03:11:16 -0400 Subject: [PATCH 3/6] [wasm] set EMSDK_PATH for makefile run-tests- targets --- src/mono/wasm/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mono/wasm/Makefile b/src/mono/wasm/Makefile index 4777abfb104c00..b9b3381f868c23 100644 --- a/src/mono/wasm/Makefile +++ b/src/mono/wasm/Makefile @@ -177,14 +177,14 @@ build-tasks: $(DOTNET) build $(TOP)/src/tasks/WasmBuildTasks $(MSBUILD_ARGS) run-tests-v8-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=V8 $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=V8 $(MSBUILD_ARGS) run-tests-sm-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=SpiderMonkey $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=SpiderMonkey $(MSBUILD_ARGS) run-tests-jsc-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=JavaScriptCore $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=JavaScriptCore $(MSBUILD_ARGS) run-tests-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) run-build-tests: PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/tests/BuildWasmApps/Wasm.Build.Tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) From 8c98f299fe861a3a9d2532bd6735ef41aa51fc51 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 21:59:21 -0400 Subject: [PATCH 4/6] [wasm] Update target dependencies for wasm --- eng/testing/tests.mobile.targets | 3 ++- eng/testing/tests.wasm.targets | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index a0e795f8f7c82b..efe67cb31648ad 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -9,6 +9,7 @@ true true true + BundleTestAppleApp;BundleTestAndroidApp @@ -254,7 +255,7 @@ + DependsOnTargets="Publish;$(BundleTestAppTargets);ArchiveTests" /> diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 7a084957d5112e..6543a871d2b706 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -1,6 +1,8 @@ + $(BundleTestAppTargets);BundleTestWasmApp + V8 $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 From 4242998b933bc7f436d7b01d0c21f10d78ad4b4d Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sun, 4 Apr 2021 01:20:14 -0400 Subject: [PATCH 5/6] Don't overwrite RunScriptCommand, thus fixing wasm build tests --- eng/testing/tests.wasm.targets | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 6543a871d2b706..505b3a8bde603a 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -2,7 +2,9 @@ $(BundleTestAppTargets);BundleTestWasmApp + + V8 $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 From e1af083803457cedb829ea77cc4ee216da141dbf Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sun, 4 Apr 2021 01:23:16 -0400 Subject: [PATCH 6/6] Track property name changes --- .../System.IO.FileSystem.Net5Compat.Tests.csproj | 2 +- .../tests/System.IO.FileSystem.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj index 51daca94299737..1389ee3462649e 100644 --- a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj @@ -5,7 +5,7 @@ $(NetCoreAppCurrent)-windows - --working-dir=/test-dir + --working-dir=/test-dir diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 97b50363adb7c9..f22b4500be176b 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -4,7 +4,7 @@ true $(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser - --working-dir=/test-dir + --working-dir=/test-dir