Cooperative request intercepts (#2403)

* some progress

* cr

* cr

* requestId is string

* unflake test

* prettier

lib/PuppeteerSharp.Tests/RequestInterceptionExperimentalTests/RequestContinueTests.cs

@@ -0,0 +1,127 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using PuppeteerSharp.Tests.Attributes;
+using PuppeteerSharp.Nunit;
+using NUnit.Framework;
+
+namespace PuppeteerSharp.Tests.RequestInterceptionExperimentalTests;
+
+public class RequestContinueTests : PuppeteerPageBaseTest
+{
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.continue", "should work")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldWork()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(async request => await request.ContinueAsync(new Payload(), 0));
+        await Page.GoToAsync(TestConstants.EmptyPage);
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.continue", "should amend HTTP headers")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldAmendHTTPHeaders()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request =>
+        {
+            var headers = new Dictionary<string, string>(request.Headers) { ["FOO"] = "bar" };
+            return request.ContinueAsync(new Payload { Headers = headers }, 0);
+        });
+        await Page.GoToAsync(TestConstants.EmptyPage);
+        var requestTask = Server.WaitForRequest("/sleep.zzz", request => request.Headers["foo"].ToString());
+        await Task.WhenAll(
+            requestTask,
+            Page.EvaluateExpressionAsync("fetch('/sleep.zzz')")
+        );
+        Assert.AreEqual("bar", requestTask.Result);
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.continue",
+        "should redirect in a way non-observable to page")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldRedirectInAWayNonObservableToPage()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request =>
+        {
+            var redirectURL = request.Url.Contains("/empty.html")
+                ? TestConstants.ServerUrl + "/consolelog.html"
+                : null;
+            return request.ContinueAsync(new Payload { Url = redirectURL }, 0);
+        });
+        string consoleMessage = null;
+        Page.Console += (_, e) => consoleMessage = e.Message.Text;
+        await Page.GoToAsync(TestConstants.EmptyPage);
+        Assert.AreEqual(TestConstants.EmptyPage, Page.Url);
+        Assert.AreEqual("yellow", consoleMessage);
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.continue", "should amend method")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldAmendMethodData()
+    {
+        await Page.GoToAsync(TestConstants.EmptyPage);
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request => request.ContinueAsync(new Payload { Method = HttpMethod.Post }, 0));
+
+        var requestTask = Server.WaitForRequest<string>("/sleep.zzz", request => request.Method);
+
+        await Task.WhenAll(
+            requestTask,
+            Page.EvaluateExpressionAsync("fetch('/sleep.zzz')")
+        );
+
+        Assert.AreEqual("POST", requestTask.Result);
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.continue", "should amend post data")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldAmendPostData()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request => request.ContinueAsync(new Payload { Method = HttpMethod.Post, PostData = "doggo" }, 0));
+        var requestTask = Server.WaitForRequest("/sleep.zzz", async request =>
+        {
+            using var reader = new StreamReader(request.Body, Encoding.UTF8);
+            return await reader.ReadToEndAsync();
+        });
+
+        await Task.WhenAll(
+            requestTask,
+            Page.GoToAsync(TestConstants.ServerUrl + "/sleep.zzz")
+        );
+
+        Assert.AreEqual("doggo", await requestTask.Result);
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.continue",
+        "should amend both post data and method on navigation")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldAmendBothPostDataAndMethodOnNavigation()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request => request.ContinueAsync(
+            new Payload
+            {
+                Method = HttpMethod.Post, PostData = "doggo"
+            },
+            0));
+
+        var serverRequestTask = Server.WaitForRequest("/empty.html", async req =>
+        {
+            var body = await new StreamReader(req.Body).ReadToEndAsync();
+            return new { req.Method, Body = body };
+        });
+
+        await Task.WhenAll(
+            serverRequestTask,
+            Page.GoToAsync(TestConstants.EmptyPage)
+        );
+        var serverRequest = await serverRequestTask;
+        Assert.AreEqual(HttpMethod.Post.Method, serverRequest.Result.Method);
+        Assert.AreEqual("doggo", serverRequest.Result.Body);
+    }
+}

lib/PuppeteerSharp.Tests/RequestInterceptionExperimentalTests/RequestRespondTests.cs

@@ -0,0 +1,149 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+using PuppeteerSharp.Tests.Attributes;
+using PuppeteerSharp.Nunit;
+using NUnit.Framework;
+
+namespace PuppeteerSharp.Tests.RequestInterceptionExperimentalTests;
+
+public class RequestRespondTests : PuppeteerPageBaseTest
+{
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.respond", "should work")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldWork()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request => request.RespondAsync(new ResponseData
+        {
+            Status = HttpStatusCode.Created,
+            Headers = new Dictionary<string, object> { ["foo"] = "bar" },
+            Body = "Yo, page!"
+        }, 0));
+
+        var response = await Page.GoToAsync(TestConstants.EmptyPage);
+        Assert.AreEqual(HttpStatusCode.Created, response.Status);
+        Assert.AreEqual("bar", response.Headers["foo"]);
+        Assert.AreEqual("Yo, page!", await Page.EvaluateExpressionAsync<string>("document.body.textContent"));
+    }
+
+    /// <summary>
+    /// In puppeteer this method is called ShouldWorkWithStatusCode422.
+    /// I found that status 422 is not available in all .NET runtimes (see https://github.com/dotnet/core/blob/4c4642d548074b3fbfd425541a968aadd75fea99/release-notes/2.1/Preview/api-diff/preview2/2.1-preview2_System.Net.md)
+    /// As the goal here is testing HTTP codes that are not in Chromium (see https://cs.chromium.org/chromium/src/net/http/http_status_code_list.h?sq=package:chromium&g=0) we will use code 426: Upgrade Required
+    /// </summary>
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.respond", "should work with status code 422")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldWorkReturnStatusPhrases()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request => request.RespondAsync(new ResponseData
+        {
+            Status = HttpStatusCode.UpgradeRequired, Body = "Yo, page!"
+        }, 0));
+
+        var response = await Page.GoToAsync(TestConstants.EmptyPage);
+        Assert.AreEqual(HttpStatusCode.UpgradeRequired, response.Status);
+        Assert.AreEqual("Upgrade Required", response.StatusText);
+        Assert.AreEqual("Yo, page!", await Page.EvaluateExpressionAsync<string>("document.body.textContent"));
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.respond", "should redirect")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldRedirect()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+
+        Page.AddRequestInterceptor(request =>
+        {
+            if (!request.Url.Contains("rrredirect"))
+            {
+                return request.ContinueAsync(new Payload(), 0);
+            }
+
+            return request.RespondAsync(new ResponseData
+            {
+                Status = HttpStatusCode.Redirect,
+                Headers = new Dictionary<string, object> { ["location"] = TestConstants.EmptyPage }
+            }, 0);
+        });
+
+        var response = await Page.GoToAsync(TestConstants.ServerUrl + "/rrredirect");
+
+        Assert.That(response.Request.RedirectChain, Has.Exactly(1).Items);
+        Assert.AreEqual(TestConstants.ServerUrl + "/rrredirect", response.Request.RedirectChain[0].Url);
+        Assert.AreEqual(TestConstants.EmptyPage, response.Url);
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.respond", "should allow mocking binary responses")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldAllowMockingBinaryResponses()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request =>
+        {
+            var imageData = File.ReadAllBytes("./Assets/pptr.png");
+            return request.RespondAsync(new ResponseData { ContentType = "image/png", BodyData = imageData }, 0);
+        });
+
+        await Page.EvaluateFunctionAsync(@"PREFIX =>
+            {
+                const img = document.createElement('img');
+                img.src = PREFIX + '/does-not-exist.png';
+                document.body.appendChild(img);
+                return new Promise(fulfill => img.onload = fulfill);
+            }", TestConstants.ServerUrl);
+        var img = await Page.QuerySelectorAsync("img");
+        Assert.True(ScreenshotHelper.PixelMatch("mock-binary-response.png", await img.ScreenshotDataAsync()));
+    }
+
+    [PuppeteerTest("requestinterception-experimental.spec.ts", "Request.respond",
+        "should stringify intercepted request response headers")]
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldStringifyInterceptedRequestResponseHeaders()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request => request.RespondAsync(new ResponseData
+        {
+            Status = HttpStatusCode.OK,
+            Headers = new Dictionary<string, object> { ["foo"] = true },
+            Body = "Yo, page!"
+        }, 0));
+
+        var response = await Page.GoToAsync(TestConstants.EmptyPage);
+        Assert.AreEqual(HttpStatusCode.OK, response.Status);
+        Assert.AreEqual("True", response.Headers["foo"]);
+        Assert.AreEqual("Yo, page!", await Page.EvaluateExpressionAsync<string>("document.body.textContent"));
+    }
+
+    [Skip(SkipAttribute.Targets.Firefox)]
+    public async Task ShouldAllowMultipleInterceptedRequestResponseHeaders()
+    {
+        await Page.SetRequestInterceptionAsync(true);
+        Page.AddRequestInterceptor(request =>
+        {
+            return request.RespondAsync(new ResponseData
+            {
+                Status = HttpStatusCode.OK,
+                Headers = new Dictionary<string, object>
+                {
+                    ["foo"] = new [] { true, false },
+                    ["Set-Cookie"] = new [] { "sessionId=abcdef", "specialId=123456" }
+                },
+                Body = "Yo, page!"
+            }, 0);
+        });
+
+        var response = await Page.GoToAsync(TestConstants.EmptyPage);
+        var cookies = await Page.GetCookiesAsync(TestConstants.EmptyPage);
+
+        Assert.AreEqual(HttpStatusCode.OK, response.Status);
+        Assert.AreEqual("True\nFalse", response.Headers["foo"]);
+        Assert.AreEqual("Yo, page!", await Page.EvaluateExpressionAsync<string>("document.body.textContent"));
+        Assert.AreEqual("specialId", cookies[0].Name);
+        Assert.AreEqual("123456", cookies[0].Value);
+        Assert.AreEqual("sessionId", cookies[1].Name);
+        Assert.AreEqual("abcdef", cookies[1].Value);
+    }
+}

lib/PuppeteerSharp.Tests/RequestInterceptionTests/SetRequestInterceptionTests.cs

@@ -15,10 +15,6 @@ namespace PuppeteerSharp.Tests.RequestInterceptionTests
 {
     public class SetRequestInterceptionTests : PuppeteerPageBaseTest
     {
-        public SetRequestInterceptionTests(): base()
-        {
-        }
-
         [PuppeteerTest("requestinterception.spec.ts", "Page.setRequestInterception", "should intercept")]
         [Skip(SkipAttribute.Targets.Firefox)]
         public async Task ShouldIntercept()
@@ -51,7 +47,7 @@ public async Task ShouldIntercept()
 
         [PuppeteerTest("requestinterception.spec.ts", "Page.setRequestInterception", "should work when POST is redirected with 302")]
         [Skip(SkipAttribute.Targets.Firefox)]
-        public async Task ShouldWorkWhenPostIsEedirectedWith302()
+        public async Task ShouldWorkWhenPostIsRedirectedWith302()
         {
             Server.SetRedirect("/rredirect", "/empty.html");
             await Page.GoToAsync(TestConstants.EmptyPage);

lib/PuppeteerSharp/BrowserData/Firefox.cs

@@ -15,7 +15,7 @@ public static class Firefox
         /// </summary>
         public const string DefaultBuildId = "FIREFOX_NIGHTLY";
 
-        private static readonly Dictionary<string, string> _cachedBuildIds = new();
+        private static readonly Dictionary<string, string> _cachedBuildIds = [];
 
         internal static Task<string> GetDefaultBuildIdAsync() => ResolveBuildIdAsync(DefaultBuildId);
 

lib/PuppeteerSharp/IPage.cs

@@ -1098,8 +1098,8 @@ public interface IPage : IDisposable, IAsyncDisposable
         Task SetOfflineModeAsync(bool value);
 
         /// <summary>
-        /// Activating request interception enables <see cref="PuppeteerSharp.Request.AbortAsync(RequestAbortErrorCode)">request.AbortAsync</see>,
-        /// <see cref="PuppeteerSharp.Request.ContinueAsync(Payload)">request.ContinueAsync</see> and <see cref="PuppeteerSharp.Request.RespondAsync(ResponseData)">request.RespondAsync</see> methods.
+        /// Activating request interception enables <see cref="PuppeteerSharp.Request.AbortAsync(RequestAbortErrorCode, int?)">request.AbortAsync</see>,
+        /// <see cref="PuppeteerSharp.Request.ContinueAsync(Payload, int?)">request.ContinueAsync</see> and <see cref="PuppeteerSharp.Request.RespondAsync(ResponseData, int?)">request.RespondAsync</see> methods.
         /// </summary>
         /// <returns>The request interception task.</returns>
         /// <param name="value">Whether to enable request interception..</param>
@@ -1438,5 +1438,19 @@ public interface IPage : IDisposable, IAsyncDisposable
         /// <param name="options">Optional waiting parameters.</param>
         /// <returns>A task that resolves after the page gets the prompt.</returns>
         Task<DeviceRequestPrompt> WaitForDevicePromptAsync(WaitTimeoutOptions options = null);
+
+        /// <summary>
+        /// <see cref="IRequest.RespondAsync"/>, <see cref="IRequest.AbortAsync"/>, and <see cref="IRequest.ContinueAsync"/> can accept an optional `priority` to activate Cooperative Intercept Mode.
+        /// In Cooperative Mode, all interception tasks are guaranteed to run and all async handlers are awaited.
+        /// The interception is resolved to the highest-priority resolution.
+        /// </summary>
+        /// <param name="interceptionTask">Interception task.</param>
+        void AddRequestInterceptor(Func<IRequest, Task> interceptionTask);
+
+        /// <summary>
+        /// Removes a previously added request interceptor.
+        /// </summary>
+        /// <param name="interceptionTask">Interception task.</param>
+        void RemoveRequestInterceptor(Func<IRequest, Task> interceptionTask);
     }
 }

lib/PuppeteerSharp/IRequest.cs

@@ -105,6 +105,11 @@ public interface IRequest
         /// <value>The redirect chain.</value>
         IRequest[] RedirectChain { get; }
 
+        /// <summary>
+        /// Information about the request initiator.
+        /// </summary>
+        public Initiator Initiator { get; }
+
         /// <summary>
         /// True when the request has POST data. Note that <see cref="PostData"/> might still be null when this flag is true
         /// when the data is too long or not readily available in the decoded form.
@@ -117,23 +122,29 @@ public interface IRequest
         /// If the URL is set it won't perform a redirect. The request will be silently forwarded to the new url. For example, the address bar will show the original url.
         /// </summary>
         /// <param name="payloadOverrides">Optional request overwrites.</param>
+        /// <param name="priority">Optional intercept abort priority. If provided, intercept will be resolved using cooperative handling rules. Otherwise, intercept will be resolved immediately.
+        /// IMPORTANT: If you set the priority, you will need to attach Request listener using <see cref="IPage.AddRequestInterceptor"/> instead of <see cref="IPage.Request"/>.</param>
         /// <returns>Task.</returns>
-        Task ContinueAsync(Payload payloadOverrides = null);
+        Task ContinueAsync(Payload payloadOverrides = null, int? priority = null);
 
         /// <summary>
         /// Fulfills request with given response. To use this, request interception should be enabled with <see cref="IPage.SetRequestInterceptionAsync(bool)"/>. Exception is thrown if request interception is not enabled.
         /// </summary>
         /// <param name="response">Response that will fulfill this request.</param>
+        /// <param name="priority">Optional intercept abort priority. If provided, intercept will be resolved using cooperative handling rules. Otherwise, intercept will be resolved immediately.
+        /// IMPORTANT: If you set the priority, you will need to attach Request listener using <see cref="IPage.AddRequestInterceptor"/> instead of <see cref="IPage.Request"/>.</param>
         /// <returns>Task.</returns>
-        Task RespondAsync(ResponseData response);
+        Task RespondAsync(ResponseData response, int? priority = null);
 
         /// <summary>
         /// Aborts request. To use this, request interception should be enabled with <see cref="IPage.SetRequestInterceptionAsync(bool)"/>.
         /// Exception is immediately thrown if the request interception is not enabled.
         /// </summary>
         /// <param name="errorCode">Optional error code. Defaults to <see cref="RequestAbortErrorCode.Failed"/>.</param>
+        /// <param name="priority">Optional intercept abort priority. If provided, intercept will be resolved using cooperative handling rules. Otherwise, intercept will be resolved immediately.
+        /// IMPORTANT: If you set the priority, you will need to attach Request listener using <see cref="IPage.AddRequestInterceptor"/> instead of <see cref="IPage.Request"/>.</param>
         /// <returns>Task.</returns>
-        Task AbortAsync(RequestAbortErrorCode errorCode = RequestAbortErrorCode.Failed);
+        Task AbortAsync(RequestAbortErrorCode errorCode = RequestAbortErrorCode.Failed, int? priority = null);
 
         /// <summary>
         /// Fetches the POST data for the request from the browser.