Skip to content

Commit

Permalink
[tests] option to include/exclude categories from Test APKs
Browse files Browse the repository at this point in the history
In some QA needs to run tests on devices without a network connection,
so an option to disable tests that rely on the network in Mono.Android-Tests
is needed.

NUnit has the option to include or exclude tests based on the
`CategoryAttribute`. Various tests were decorated with
`[Category("InetAccess")]` that appeared to use the internet. To verify
this works, I ran the tests with my internet disabled. The test run for
Mono.Android-Tests went down from 106 to 84 tests, with no failures.

To exclude a category, on Windows:
    msbuild Xamarin.Android.sln /t:RunApkTests /p:ExcludeCategories=InetAccess

On MacOS/Linux:
    make run-apk-tests EXCLUDECATEGORIES=InetAccess

If you want to specify multiple categories, use `,` (comma) to delimit
each category. This was what NUnit uses to delimit categories as per
their documentation here: http://nunit.org/docs/2.6/consoleCommandLine.html

I also added the option specify `Include` as well, but it isn't particularly
useful for this case.
  • Loading branch information
jonathanpeppers committed Nov 14, 2017
1 parent 19bd3de commit e58c5c5
Show file tree
Hide file tree
Showing 10 changed files with 46 additions and 11 deletions.
14 changes: 14 additions & 0 deletions Documentation/DevelopmentTips.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,20 @@ For example:
$ tools/scripts/xabuild /t:DeployTestApks tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
$ tools/scripts/xabuild /t:RunTestApks tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj

## Running `.apk` Projects with Include/Exclude

For example, to exclude tests that use the internet (`InetAccess` category):

$ make run-apk-tests EXCLUDECATEGORIES=InetAccess

On Windows:

$ msbuild Xamarin.Android.sln /t:RunApkTests /p:ExcludeCategories=InetAccess

`INCLUDECATEGORIES` and `IncludeCategories` function in the same fashion.

To specify multiple categories, delimit each category with a `,` character.

### Running A Single Test Fixture

A single NUnit *Test Fixture* -- a class with the `[TestFixture]`
Expand Down
5 changes: 5 additions & 0 deletions build-tools/scripts/TestApks.targets
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,17 @@

<Target Name="RunTestApks"
Condition=" '@(TestApk)' != '' ">
<PropertyGroup>
<_IncludeCategories Condition=" '$(IncludeCategories)' != '' ">include=$(IncludeCategories)</_IncludeCategories>
<_ExcludeCategories Condition=" '$(ExcludeCategories)' != '' ">exclude=$(ExcludeCategories)</_ExcludeCategories>
</PropertyGroup>
<RunInstrumentationTests
Condition=" '%(TestApk.InstrumentationType)' != ''"
AdbTarget="$(_AdbTarget)"
AdbOptions="$(AdbOptions)"
Component="%(TestApk.Package)/%(TestApk.InstrumentationType)"
NUnit2TestResultsFile="%(TestApk.ResultsPath)"
InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories)"
TestFixture="$(TestFixture)"
ToolExe="$(AdbToolExe)"
ToolPath="$(AdbToolPath)">
Expand Down
2 changes: 1 addition & 1 deletion src/Mono.Android/Test/System.Net/ProxyTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

namespace System.NetTests {

[TestFixture]
[TestFixture, Category ("InetAccess")]
public class ProxyTest {

// https://bugzilla.xamarin.com/show_bug.cgi?id=14968
Expand Down
2 changes: 1 addition & 1 deletion src/Mono.Android/Test/System.Net/SslTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

namespace System.NetTests {

[TestFixture]
[TestFixture, Category ("InetAccess")]
public class SslTest {

// https://xamarin.desk.com/agent/case/35534
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@
using Android.OS;

namespace Xamarin.Android.NetTests {

[Category("InetAccess")]
public abstract class HttpClientHandlerTestBase
{
protected abstract HttpClientHandler CreateHandler ();
protected abstract HttpClientHandler CreateHandler ();

class Proxy : IWebProxy
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
using System.IO;

namespace Xamarin.Android.NetTests {

[Category ("InetAccess")]
public abstract class HttpClientIntegrationTestBase
{
protected abstract HttpClientHandler CreateHandler ();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,12 @@ protected override void OnResume ()

protected virtual IEnumerable <string> GetIncludedCategories ()
{
return null;
yield break;
}

protected virtual IEnumerable <string> GetExcludedCategories ()
{
return null;
yield break;
}

// Subclasses can override this method to update the test filtering that the runner will use.
Expand Down
3 changes: 2 additions & 1 deletion src/Xamarin.Android.NUnitLite/Gui/AndroidRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,8 @@ static void ChainCategoryFilter (IEnumerable <string> categories, bool negate, r
gotCategories = true;
}

chain = new AndFilter (chain, negate ? (TestFilter)new NotFilter (filter) : (TestFilter)filter);
if (gotCategories)
chain = new AndFilter (chain, negate ? (TestFilter)new NotFilter (filter) : (TestFilter)filter);
}

if (!gotCategories)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,12 +146,22 @@ protected void AddTest (Assembly assembly)

protected virtual IEnumerable <string> GetIncludedCategories ()
{
return null;
string include = arguments?.GetString ("include");
if (!string.IsNullOrEmpty (include)) {
foreach (var category in include.Split (',')) {
yield return category;
}
}
}

protected virtual IEnumerable <string> GetExcludedCategories ()
{
return null;
string exclude = arguments?.GetString ("exclude");
if (!string.IsNullOrEmpty (exclude)) {
foreach (var category in exclude.Split (',')) {
yield return category;
}
}
}

protected virtual void UpdateFilter ()
Expand Down
7 changes: 6 additions & 1 deletion tests/Xamarin.Android.Bcl-Tests/TestInstrumentation.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,12 @@ protected override void AddTests ()

protected override IEnumerable<string> GetExcludedCategories ()
{
return App.GetExcludedCategories ();
foreach (var category in base.GetExcludedCategories ()) {
yield return category;
}
foreach (var category in App.GetExcludedCategories ()) {
yield return category;
}
}

protected override void UpdateFilter ()
Expand Down

0 comments on commit e58c5c5

Please sign in to comment.