Replace lone surrogates option (#2780)

kblok · web-flow · commit a22adb0761ad · 2024-09-12T18:45:21.000-03:00
diff --git a/lib/PuppeteerSharp.TestServer/wwwroot/lone-surrogate.html b/lib/PuppeteerSharp.TestServer/wwwroot/lone-surrogate.html
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Lone Surrogate via JavaScript</title>
+</head>
+<body>
+<h1>Lone Surrogate Example</h1>
+<p id="surrogate-text">This paragraph will be updated with a lone surrogate.</p>
+
+<script>
+    // Create a lone surrogate
+    var loneSurrogate = String.fromCharCode(0xD83D); // High surrogate without its pair
+    document.getElementById('surrogate-text').textContent = "This paragraph contains a lone surrogate: " + loneSurrogate;
+</script>
+</body>
+</html>
diff --git a/lib/PuppeteerSharp.Tests/PageTests/GetContentTests.cs b/lib/PuppeteerSharp.Tests/PageTests/GetContentTests.cs
@@ -0,0 +1,20 @@
+using System;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using PuppeteerSharp.Nunit;
+
+namespace PuppeteerSharp.Tests.PageTests
+{
+    public class GetContentTests : PuppeteerPageBaseTest
+    {
+
+        [Test, Retry(2)]
+        public async Task ShouldWorkWithLoneSurrogate()
+        {
+            await Page.GoToAsync(TestConstants.ServerUrl + "/lone-surrogate.html");
+            var result = await Page.GetContentAsync(new GetContentOptions { ReplaceLoneSurrogates = true });
+
+            Assert.That(result, Contains.Substring("This paragraph contains a lone surrogate"));
+        }
+    }
+}
diff --git a/lib/PuppeteerSharp/Frame.cs b/lib/PuppeteerSharp/Frame.cs
@@ -205,22 +205,24 @@ public Task<DeviceRequestPrompt> WaitForDevicePromptAsync(WaitForOptions options
         public abstract Task<IElementHandle> AddScriptTagAsync(AddTagOptions options);
 
         /// <inheritdoc/>
-        public Task<string> GetContentAsync()
-            => EvaluateFunctionAsync<string>(@"() => {
-                let content = '';
-                for (const node of document.childNodes) {
-                    switch (node) {
-                    case document.documentElement:
-                        content += document.documentElement.outerHTML;
-                        break;
-                    default:
-                        content += new XMLSerializer().serializeToString(node);
-                        break;
+        public Task<string> GetContentAsync(GetContentOptions options = null)
+            => EvaluateFunctionAsync<string>(
+                @"(replaceLoneSurrogates) => {
+                    let content = '';
+                    for (const node of document.childNodes) {
+                        switch (node) {
+                        case document.documentElement:
+                            content += document.documentElement.outerHTML;
+                            break;
+                        default:
+                            content += new XMLSerializer().serializeToString(node);
+                            break;
+                        }
                     }
-                }
 
-                return content;
-            }");
+                    return replaceLoneSurrogates ? content.toWellFormed() : content;
+                }",
+                options?.ReplaceLoneSurrogates ?? false);
 
         /// <inheritdoc/>
         public abstract Task SetContentAsync(string html, NavigationOptions options = null);
diff --git a/lib/PuppeteerSharp/GetContentOptions.cs b/lib/PuppeteerSharp/GetContentOptions.cs
@@ -0,0 +1,38 @@
+// * MIT License
+//  *
+//  * Copyright (c) Darío Kondratiuk
+//  *
+//  * Permission is hereby granted, free of charge, to any person obtaining a copy
+//  * of this software and associated documentation files (the "Software"), to deal
+//  * in the Software without restriction, including without limitation the rights
+//  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  * copies of the Software, and to permit persons to whom the Software is
+//  * furnished to do so, subject to the following conditions:
+//  *
+//  * The above copyright notice and this permission notice shall be included in all
+//  * copies or substantial portions of the Software.
+//  *
+//  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+//  * SOFTWARE.
+
+namespace PuppeteerSharp;
+
+/// <summary>
+/// Options for <see cref="IPage.GetContentAsync(GetContentOptions)"/> and <see cref="IFrame.GetContentAsync(GetContentOptions)"/>.
+/// </summary>
+public class GetContentOptions
+{
+    /// <summary>
+    /// Replace lone surrogates with U+FFFD replacement character.
+    /// </summary>
+    /// <remarks>
+    /// This functionality relies on the toWellFormed function. <a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/toWellFormed">See.</a>.
+    /// It's set to false by default to prevent extra processing when not needed.
+    /// </remarks>
+    public bool ReplaceLoneSurrogates { get; set; } = false;
+}
diff --git a/lib/PuppeteerSharp/IFrame.cs b/lib/PuppeteerSharp/IFrame.cs
@@ -210,9 +210,10 @@ public interface IFrame
         /// <summary>
         /// Gets the full HTML contents of the page, including the doctype.
         /// </summary>
+        /// <param name="options">Options.</param>
         /// <returns>Task which resolves to the HTML content.</returns>
         /// <seealso cref="IPage.GetContentAsync"/>
-        Task<string> GetContentAsync();
+        Task<string> GetContentAsync(GetContentOptions options = null);
 
         /// <summary>
         /// Returns page's title.
diff --git a/lib/PuppeteerSharp/IPage.cs b/lib/PuppeteerSharp/IPage.cs
@@ -749,9 +749,10 @@ public interface IPage : IDisposable, IAsyncDisposable
         /// <summary>
         /// Gets the full HTML contents of the page, including the doctype.
         /// </summary>
+        /// <param name="options">Options.</param>
         /// <returns>Task which resolves to the HTML content.</returns>
         /// <seealso cref="IFrame.GetContentAsync"/>
-        Task<string> GetContentAsync();
+        Task<string> GetContentAsync(GetContentOptions options = null);
 
         /// <summary>
         /// Returns the page's cookies.
diff --git a/lib/PuppeteerSharp/Page.cs b/lib/PuppeteerSharp/Page.cs
@@ -349,7 +349,7 @@ public Task ExposeFunctionAsync<T1, T2, T3, T4, TResult>(
         public abstract Task RemoveExposedFunctionAsync(string name);
 
         /// <inheritdoc/>
-        public Task<string> GetContentAsync() => MainFrame.GetContentAsync();
+        public Task<string> GetContentAsync(GetContentOptions options = null) => MainFrame.GetContentAsync(options);
 
         /// <inheritdoc/>
         public Task SetContentAsync(string html, NavigationOptions options = null)
