Skip to content

Commit

Permalink
chore(playwrighttesting): add logging per framework and fix auth hand…
Browse files Browse the repository at this point in the history
…ling in vstest (#47180)

* chore(playwrighttesting): add logging for nunit and vstest

* chore(): update public api docs

---------

Co-authored-by: Siddharth Singha Roy <[email protected]>
  • Loading branch information
Sid200026 and Siddharth Singha Roy authored Nov 15, 2024
1 parent 7670deb commit 1c7b970
Show file tree
Hide file tree
Showing 10 changed files with 201 additions and 53 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit
[NUnit.Framework.SetUpFixtureAttribute]
public partial class PlaywrightServiceNUnit : Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightService
{
public PlaywrightServiceNUnit(Azure.Core.TokenCredential? credential = null) : base (default(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions), default(Azure.Core.TokenCredential)) { }
public PlaywrightServiceNUnit(Azure.Core.TokenCredential? credential = null) : base (default(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions), default(Azure.Core.TokenCredential), default(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface.IFrameworkLogger)) { }
public static Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions playwrightServiceOptions { get { throw null; } }
[NUnit.Framework.OneTimeSetUpAttribute]
public System.Threading.Tasks.Task SetupAsync() { throw null; }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
using NUnit.Framework;

namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit
{
internal class NUnitFrameworkLogger : IFrameworkLogger
{
public void Debug(string message)
{
TestContext.WriteLine($"[MPT-NUnit]: {message}");
}

public void Error(string message)
{
TestContext.Error.WriteLine($"[MPT-NUnit]: {message}");
}

public void Info(string message)
{
TestContext.Progress.WriteLine($"[MPT-NUnit]: {message}");
}

public void Warning(string message)
{
TestContext.Progress.WriteLine($"[MPT-NUnit]: {message}");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System.Runtime.InteropServices;
using System;
using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger;
using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;

namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit;

Expand All @@ -16,12 +17,13 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.NUnit;
[SetUpFixture]
public class PlaywrightServiceNUnit : PlaywrightService
{
private static NUnitFrameworkLogger nunitFrameworkLogger { get; } = new();
/// <summary>
/// Initializes a new instance of the <see cref="PlaywrightServiceNUnit"/> class.
/// </summary>
/// <param name="credential">The azure token credential to use for authentication.</param>
public PlaywrightServiceNUnit(TokenCredential? credential = null)
: base(playwrightServiceOptions, credential: credential)
: base(playwrightServiceOptions, credential: credential, frameworkLogger: nunitFrameworkLogger)
{
}

Expand All @@ -47,7 +49,7 @@ public async Task SetupAsync()
{
if (!UseCloudHostedBrowsers)
return;
TestContext.Progress.WriteLine("\nRunning tests using Microsoft Playwright Testing service.\n");
nunitFrameworkLogger.Info("\nRunning tests using Microsoft Playwright Testing service.\n");

await InitializeAsync().ConfigureAwait(false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ public ConnectOptions() { }
}
public partial class PlaywrightService
{
public PlaywrightService(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions playwrightServiceOptions, Azure.Core.TokenCredential? credential = null) { }
public PlaywrightService(System.Runtime.InteropServices.OSPlatform? os = default(System.Runtime.InteropServices.OSPlatform?), string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = default(bool?), Azure.Core.TokenCredential? credential = null) { }
public PlaywrightService(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.PlaywrightServiceOptions playwrightServiceOptions, Azure.Core.TokenCredential? credential = null, Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface.IFrameworkLogger? frameworkLogger = null) { }
public PlaywrightService(System.Runtime.InteropServices.OSPlatform? os = default(System.Runtime.InteropServices.OSPlatform?), string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = default(bool?), Azure.Core.TokenCredential? credential = null, Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface.IFrameworkLogger? frameworkLogger = null) { }
public string? ExposeNetwork { get { throw null; } set { } }
public System.Runtime.InteropServices.OSPlatform? Os { get { throw null; } set { } }
public System.Threading.Timer? RotationTimer { get { throw null; } set { } }
Expand Down Expand Up @@ -67,3 +67,13 @@ public enum ServiceVersion
}
}
}
namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface
{
public partial interface IFrameworkLogger
{
void Debug(string message);
void Error(string message);
void Info(string message);
void Warning(string message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Threading;
using System.Threading.Tasks;
using Azure.Core;
using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
using Azure.Identity;
using Microsoft.IdentityModel.JsonWebTokens;

Expand All @@ -16,9 +17,11 @@ internal class EntraLifecycle
internal long? _entraIdAccessTokenExpiry;
private readonly TokenCredential _tokenCredential;
private readonly JsonWebTokenHandler _jsonWebTokenHandler;
private readonly IFrameworkLogger? _frameworkLogger;

public EntraLifecycle(TokenCredential? tokenCredential = null, JsonWebTokenHandler? jsonWebTokenHandler = null)
public EntraLifecycle(TokenCredential? tokenCredential = null, JsonWebTokenHandler? jsonWebTokenHandler = null, IFrameworkLogger? frameworkLogger = null)
{
_frameworkLogger = frameworkLogger;
_tokenCredential = tokenCredential ?? new DefaultAzureCredential();
_jsonWebTokenHandler = jsonWebTokenHandler ?? new JsonWebTokenHandler();
SetEntraIdAccessTokenFromEnvironment();
Expand All @@ -37,7 +40,7 @@ internal async Task FetchEntraIdAccessTokenAsync(CancellationToken cancellationT
}
catch (Exception ex)
{
Console.Error.WriteLine(ex);
_frameworkLogger?.Error(ex.ToString());
throw new Exception(Constants.s_no_auth_error);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;

namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation
{
internal class VSTestFrameworkLogger : IFrameworkLogger
{
private readonly ILogger _logger;
public VSTestFrameworkLogger(ILogger? logger = null)
{
_logger = logger ?? new Logger();
}

public void Debug(string message)
{
_logger.Debug(message);
}

public void Error(string message)
{
_logger.Error(message);
}

public void Info(string message)
{
_logger.Info(message);
}

public void Warning(string message)
{
_logger.Warning(message);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface
{
/// <summary>
/// Sets up logging for the TestLogger package.
/// </summary>
public interface IFrameworkLogger
{
/// <summary>
/// Log informational message.
/// </summary>
/// <param name="message"></param>
void Info(string message);
/// <summary>
/// Log debug messages.
/// </summary>
/// <param name="message"></param>
void Debug(string message);
/// <summary>
/// Log warnming messages.
/// </summary>
/// <param name="message"></param>
void Warning(string message);
/// <summary>
/// Log error messages.
/// </summary>
/// <param name="message"></param>
void Error(string message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,19 @@ internal void InitializePlaywrightReporter(string xmlSettings)
return;
}
// setup entra rotation handlers
_playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, entraLifecycle: null, jsonWebTokenHandler: _jsonWebTokenHandler, credential: playwrightServiceSettings.AzureTokenCredential);
IFrameworkLogger frameworkLogger = new VSTestFrameworkLogger(_logger);
try
{
_playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, entraLifecycle: null, jsonWebTokenHandler: _jsonWebTokenHandler, credential: playwrightServiceSettings.AzureTokenCredential, frameworkLogger: frameworkLogger);
#pragma warning disable AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead.
_playwrightService.InitializeAsync().GetAwaiter().GetResult();
_playwrightService.InitializeAsync().GetAwaiter().GetResult();
#pragma warning restore AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead.
}
catch (Exception ex)
{
// We have checks for access token and base url in the next block, so we can ignore the exception here.
_logger.Error("Failed to initialize PlaywrightService: " + ex);
}

var cloudRunId = _environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId);
string baseUrl = _environment.GetEnvironmentVariable(ReporterConstants.s_pLAYWRIGHT_SERVICE_REPORTING_URL);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT License.

using Azure.Core;
using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface;
using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model;
using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility;
using Microsoft.IdentityModel.JsonWebTokens;
Expand Down Expand Up @@ -136,19 +137,22 @@ public string? ExposeNetwork

private readonly EntraLifecycle? _entraLifecycle;
private readonly JsonWebTokenHandler? _jsonWebTokenHandler;
private IFrameworkLogger? _frameworkLogger;

/// <summary>
/// Initializes a new instance of the <see cref="PlaywrightService"/> class.
/// </summary>
/// <param name="playwrightServiceOptions"></param>
/// <param name="credential"></param>
public PlaywrightService(PlaywrightServiceOptions playwrightServiceOptions, TokenCredential? credential = null) : this(
/// <param name="frameworkLogger"></param>
public PlaywrightService(PlaywrightServiceOptions playwrightServiceOptions, TokenCredential? credential = null, IFrameworkLogger? frameworkLogger = null) : this(
os: playwrightServiceOptions.Os,
runId: playwrightServiceOptions.RunId,
exposeNetwork: playwrightServiceOptions.ExposeNetwork,
serviceAuth: playwrightServiceOptions.ServiceAuth,
useCloudHostedBrowsers: playwrightServiceOptions.UseCloudHostedBrowsers,
credential: credential ?? playwrightServiceOptions.AzureTokenCredential
credential: credential ?? playwrightServiceOptions.AzureTokenCredential,
frameworkLogger: frameworkLogger
)
{
// No-op
Expand All @@ -163,21 +167,25 @@ public PlaywrightService(PlaywrightServiceOptions playwrightServiceOptions, Toke
/// <param name="serviceAuth">The service authentication mechanism.</param>
/// <param name="useCloudHostedBrowsers">Whether to use cloud-hosted browsers.</param>
/// <param name="credential">The token credential.</param>
public PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, TokenCredential? credential = null)
/// <param name="frameworkLogger">Logger</param>
public PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, TokenCredential? credential = null, IFrameworkLogger? frameworkLogger = null)
{
if (string.IsNullOrEmpty(ServiceEndpoint))
return;
_entraLifecycle = new EntraLifecycle(tokenCredential: credential);
_frameworkLogger = frameworkLogger;
_entraLifecycle = new EntraLifecycle(tokenCredential: credential, frameworkLogger: _frameworkLogger);
_jsonWebTokenHandler = new JsonWebTokenHandler();
InitializePlaywrightServiceEnvironmentVariables(GetServiceCompatibleOs(os), runId, exposeNetwork, serviceAuth, useCloudHostedBrowsers);
}

internal PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, EntraLifecycle? entraLifecycle = null, JsonWebTokenHandler? jsonWebTokenHandler = null, TokenCredential? credential = null)
internal PlaywrightService(OSPlatform? os = null, string? runId = null, string? exposeNetwork = null, string? serviceAuth = null, bool? useCloudHostedBrowsers = null, EntraLifecycle? entraLifecycle = null, JsonWebTokenHandler? jsonWebTokenHandler = null, TokenCredential? credential = null, IFrameworkLogger? frameworkLogger = null)
{
if (string.IsNullOrEmpty(ServiceEndpoint))
return;
_frameworkLogger = frameworkLogger;
_jsonWebTokenHandler = jsonWebTokenHandler ?? new JsonWebTokenHandler();
_entraLifecycle = entraLifecycle ?? new EntraLifecycle(credential, _jsonWebTokenHandler);
_entraLifecycle = entraLifecycle ?? new EntraLifecycle(credential, _jsonWebTokenHandler, _frameworkLogger);
_frameworkLogger = frameworkLogger;
InitializePlaywrightServiceEnvironmentVariables(GetServiceCompatibleOs(os), runId, exposeNetwork, serviceAuth, useCloudHostedBrowsers);
}

Expand Down Expand Up @@ -211,6 +219,7 @@ internal PlaywrightService(OSPlatform? os = null, string? runId = null, string?
}
if (string.IsNullOrEmpty(GetAuthToken()))
{
_frameworkLogger?.Error("Access token not found when trying to call GetConnectOptionsAsync.");
throw new Exception(Constants.s_no_auth_error);
}

Expand All @@ -236,20 +245,26 @@ internal PlaywrightService(OSPlatform? os = null, string? runId = null, string?
public async Task InitializeAsync(CancellationToken cancellationToken = default)
{
if (string.IsNullOrEmpty(ServiceEndpoint))
{
_frameworkLogger?.Info("Exiting initialization as service endpoint is not set.");
return;
}
if (!UseCloudHostedBrowsers)
{
// Since playwright-dotnet checks PLAYWRIGHT_SERVICE_ACCESS_TOKEN and PLAYWRIGHT_SERVICE_URL to be set, remove PLAYWRIGHT_SERVICE_URL so that tests are run locally.
// If customers use GetConnectOptionsAsync, after setting disableScalableExecution, an error will be thrown.
_frameworkLogger?.Info("Disabling scalable execution since UseCloudHostedBrowsers is set to false.");
Environment.SetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceUri, null);
return;
}
// If default auth mechanism is Access token and token is available in the environment variable, no need to setup rotation handler
if (ServiceAuth == ServiceAuthType.AccessToken)
{
_frameworkLogger?.Info("Auth mechanism is Access Token.");
ValidateMptPAT();
return;
}
_frameworkLogger?.Info("Auth mechanism is Entra Id.");
await _entraLifecycle!.FetchEntraIdAccessTokenAsync(cancellationToken).ConfigureAwait(false);
RotationTimer = new Timer(RotationHandlerAsync, null, TimeSpan.FromMinutes(Constants.s_entra_access_token_rotation_interval_period_in_minutes), TimeSpan.FromMinutes(Constants.s_entra_access_token_rotation_interval_period_in_minutes));
}
Expand All @@ -259,13 +274,15 @@ public async Task InitializeAsync(CancellationToken cancellationToken = default)
/// </summary>
public void Cleanup()
{
_frameworkLogger?.Info("Cleaning up Playwright service resources.");
RotationTimer?.Dispose();
}

internal async void RotationHandlerAsync(object? _)
{
if (_entraLifecycle!.DoesEntraIdAccessTokenRequireRotation())
{
_frameworkLogger?.Info("Rotating Entra Id access token.");
await _entraLifecycle.FetchEntraIdAccessTokenAsync().ConfigureAwait(false);
}
}
Expand Down Expand Up @@ -360,20 +377,28 @@ internal static void SetReportingUrlAndWorkspaceId()

private void ValidateMptPAT()
{
string authToken = GetAuthToken()!;
if (string.IsNullOrEmpty(authToken))
throw new Exception(Constants.s_no_auth_error);
JsonWebToken jsonWebToken = _jsonWebTokenHandler!.ReadJsonWebToken(authToken) ?? throw new Exception(Constants.s_invalid_mpt_pat_error);
var tokenWorkspaceId = jsonWebToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value;
Match match = Regex.Match(ServiceEndpoint, @"wss://(?<region>[\w-]+)\.api\.(?<domain>playwright(?:-test|-int)?\.io|playwright\.microsoft\.com)/accounts/(?<workspaceId>[\w-]+)/");
if (!match.Success)
throw new Exception(Constants.s_invalid_service_endpoint_error_message);
var serviceEndpointWorkspaceId = match.Groups["workspaceId"].Value;
if (tokenWorkspaceId != serviceEndpointWorkspaceId)
throw new Exception(Constants.s_workspace_mismatch_error);
var expiry = (long)(jsonWebToken.ValidTo - new DateTime(1970, 1, 1)).TotalSeconds;
if (expiry <= DateTimeOffset.UtcNow.ToUnixTimeSeconds())
throw new Exception(Constants.s_expired_mpt_pat_error);
try
{
string authToken = GetAuthToken()!;
if (string.IsNullOrEmpty(authToken))
throw new Exception(Constants.s_no_auth_error);
JsonWebToken jsonWebToken = _jsonWebTokenHandler!.ReadJsonWebToken(authToken) ?? throw new Exception(Constants.s_invalid_mpt_pat_error);
var tokenWorkspaceId = jsonWebToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value;
Match match = Regex.Match(ServiceEndpoint, @"wss://(?<region>[\w-]+)\.api\.(?<domain>playwright(?:-test|-int)?\.io|playwright\.microsoft\.com)/accounts/(?<workspaceId>[\w-]+)/");
if (!match.Success)
throw new Exception(Constants.s_invalid_service_endpoint_error_message);
var serviceEndpointWorkspaceId = match.Groups["workspaceId"].Value;
if (tokenWorkspaceId != serviceEndpointWorkspaceId)
throw new Exception(Constants.s_workspace_mismatch_error);
var expiry = (long)(jsonWebToken.ValidTo - new DateTime(1970, 1, 1)).TotalSeconds;
if (expiry <= DateTimeOffset.UtcNow.ToUnixTimeSeconds())
throw new Exception(Constants.s_expired_mpt_pat_error);
}
catch (Exception ex)
{
_frameworkLogger?.Error(ex.ToString());
throw;
}
}

internal static string? GetServiceCompatibleOs(OSPlatform? oSPlatform)
Expand Down
Loading

0 comments on commit 1c7b970

Please sign in to comment.