-
Notifications
You must be signed in to change notification settings - Fork 653
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Test to Scan All Pages for Axe Issues #1361
Merged
Merged
Changes from all commits
Commits
Show all changes
27 commits
Select commit
Hold shift + click to select a range
4198bb8
add uniqueId to automationIds
karkarl fe5cff0
Scall All Test setup
karkarl 718e807
update axe helper message
karkarl 8540aa9
fix iconspage test failures
karkarl 1c06872
scan all test loop
karkarl 7d3c8aa
fix test failure
karkarl 84e3761
fix test failures
karkarl 590866a
cleanup
karkarl 1dd17ac
fix warning
karkarl f77bff5
remove duplicate axe tests
karkarl 1225f10
cleanup
karkarl abb4e03
cleanup
karkarl fae754a
revert change
karkarl 8538b70
fix build error
karkarl a4af033
generate xml test data
karkarl bd51427
testContext wip
karkarl da906e8
dynamic data wip
karkarl 2d7126a
dynamic data tests
karkarl ac9a179
remove parameters from xml
karkarl 454f6f4
revert change
karkarl 68ea61a
remove allyView raw
karkarl 8b14af0
individual page test data
karkarl 8f67e24
cleanup
karkarl 145740e
remove double nesting
karkarl 2740119
exclusion list
karkarl 2223aa0
revert
karkarl 0df5a37
refactor to parse json
karkarl File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,116 @@ | ||
using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
using Microsoft.VisualStudio.TestPlatform; | ||
using OpenQA.Selenium.Appium.Windows; | ||
using System; | ||
using System.Linq; | ||
using System.Text.Json; | ||
using System.IO; | ||
using System.Collections.ObjectModel; | ||
using System.Collections; | ||
using System.Collections.Generic; | ||
using System.Xml; | ||
using System.Reflection; | ||
using Newtonsoft.Json; | ||
|
||
namespace UITests.Tests | ||
{ | ||
[TestClass] | ||
public class AxeScanAll : TestBase | ||
{ | ||
public static readonly string jsonUri = "ControlInfoData.json"; | ||
public static new WindowsDriver<WindowsElement> Session => SessionManager.Session; | ||
|
||
public static string[] ExclusionList = | ||
{ | ||
"WebView2", // 46668961: Web contents from WebView2 are throwing null BoundingRectangle errors. | ||
"Icons" // https://github.com/CommunityToolkit/Windows/issues/240 External toolkit SettingsExpander does not pass Axe testing | ||
}; | ||
|
||
public class ControlInfoData | ||
{ | ||
public List<Group> Groups { get; set; } | ||
} | ||
|
||
public class Group | ||
{ | ||
[JsonProperty("UniqueId")] | ||
public string UniqueId { get; set; } | ||
|
||
[JsonProperty("Items")] | ||
public List<Item> Items { get; set; } | ||
} | ||
|
||
public class Item | ||
{ | ||
[JsonProperty("UniqueId")] | ||
public string UniqueId { get; set; } | ||
} | ||
|
||
private static IEnumerable<object[]> TestData() | ||
{ | ||
var testCases = new List<object[]>(); | ||
|
||
string jsonContent = System.IO.File.ReadAllText(jsonUri); | ||
var controlInfoData = JsonConvert.DeserializeObject<ControlInfoData>(jsonContent); | ||
|
||
foreach (var group in controlInfoData.Groups) | ||
{ | ||
var sectionName = group.UniqueId; | ||
|
||
// Select all row names within the current table | ||
var items = group.Items; | ||
|
||
foreach (var item in items) | ||
{ | ||
var pageName = item.UniqueId; | ||
|
||
// Skip pages in the exclusion list. | ||
if (ExclusionList.Contains(pageName)) | ||
{ | ||
continue; | ||
} | ||
testCases.Add(new object[] { sectionName, pageName }); | ||
} | ||
} | ||
|
||
return testCases; | ||
} | ||
|
||
[ClassInitialize] | ||
public static void ClassInitialize(TestContext context) | ||
{ | ||
} | ||
|
||
[TestMethod] | ||
[DynamicData(nameof(TestData), DynamicDataSourceType.Method, DynamicDataDisplayName = nameof(GetCustomDynamicDataDisplayName))] | ||
[TestProperty("Description", "Scan pages in the WinUIGallery for accessibility issues.")] | ||
public void ValidatePageAccessibilityWithAxe(string sectionName, string pageName) | ||
{ | ||
try | ||
{ | ||
// Click into page and check for accessibility issues. | ||
var page = Session.FindElementByAccessibilityId(pageName); | ||
page.Click(); | ||
|
||
AxeHelper.AssertNoAccessibilityErrors(); | ||
} | ||
catch | ||
{ | ||
// If element is not found, expand tree view as it is nested. | ||
var section = Session.FindElementByAccessibilityId(sectionName); | ||
section.Click(); | ||
|
||
// Click into page and check for accessibility issues. | ||
var page = Session.FindElementByAccessibilityId(pageName); | ||
page.Click(); | ||
|
||
AxeHelper.AssertNoAccessibilityErrors(); | ||
} | ||
} | ||
|
||
public static string GetCustomDynamicDataDisplayName(MethodInfo methodInfo, object[] data) | ||
{ | ||
return string.Format("Validate{0}PageAccessibility", data[1]); | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Since you are doing your own parsing of the data, would it be simpler to directly parse the ControlInfoData.json file instead of having a separate script to convert ControlInfoData.json into WinUIGalleryTestData.xml and then parsing that xml file?
For the Lifted Xaml repo, I needed to convert from json into an xml format that taef understood. But since you re doing your own parsing here, you don't have to use xml at all. #Closed
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I believe it's easier to parse the .xml file instead of the .json. The json file has quite a lot of info that will need to be parsed, but is not relevant to the tests. This method adds an extra file, but results in cleaner test code.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What about using the same parsing code from the WinUI 3 Gallery? Since we are using the same .NET SDK, we could move them in a shared project so both projects can reference the same data and parsing.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You are already parsing the json file. You are just doing it in powershell as part of the build. You can move that same logic out of powershell and into the test code.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@kmahone Done.
@chingucoding That can definitely be a future refactor, but too big of a change for this PR in my opinion.