diff --git a/CHANGELOG.md b/CHANGELOG.md index de54c52..d2d2fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Released on Sunday, November 8 2026. - Fixed `GetAttachedFileName` passing on folders from `Content-Disposition`; it now gives a plain file name @michaelthyregod - Fixed a cookie `Expires` date with a day the month does not have failing the page load @michaelthyregod +- Fixed the cookie provider failing for good once cookies were set or read from several threads at a time, as when a page loads its resources @michaelthyregod - Fixed `RemoveCookies` returning no cookies and `RemoveAllCookies` not updating the cookie file @michaelthyregod - Added trimming and Native AOT compatibility (`IsAotCompatible`) for the .NET 8 and .NET 10 targets @michaelthyregod diff --git a/src/AngleSharp.Io.Tests/Cookie/ConcurrencyTests.cs b/src/AngleSharp.Io.Tests/Cookie/ConcurrencyTests.cs new file mode 100644 index 0000000..a80433f --- /dev/null +++ b/src/AngleSharp.Io.Tests/Cookie/ConcurrencyTests.cs @@ -0,0 +1,97 @@ +namespace AngleSharp.Io.Tests.Cookie +{ + using AngleSharp.Dom; + using AngleSharp.Io.Cookie; + using NUnit.Framework; + using System; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + [TestFixture] + public class ConcurrencyTests + { + private const Int32 Threads = 8; + private const Int32 CookiesPerThread = 200; + + [Test] + public async Task ParallelSetAndGetKeepAllCookies() + { + var handler = new MemoryFileHandler(); + var provider = new AdvancedCookieProvider(handler); + var url = new Url("https://example.com/"); + + await RunInParallel(thread => + { + for (var i = 0; i < CookiesPerThread; i++) + { + ((ICookieProvider)provider).SetCookie(url, $"k{thread}_{i}=v; Max-Age=1000"); + ((ICookieProvider)provider).GetCookie(url); + } + }).ConfigureAwait(false); + + Assert.AreEqual(Threads * CookiesPerThread, provider.Cookies.Count()); + Assert.AreEqual(Threads * CookiesPerThread, new AdvancedCookieProvider(handler).Cookies.Count()); + } + + [Test] + public async Task ParallelRemovalsRemoveEachCookieOnce() + { + var handler = new MemoryFileHandler(); + var provider = new AdvancedCookieProvider(handler); + var url = new Url("https://example.com/"); + + for (var i = 0; i < Threads * CookiesPerThread; i++) + { + ((ICookieProvider)provider).SetCookie(url, $"k{i}=v; Max-Age=1000"); + } + + var removed = 0; + + await RunInParallel(thread => + { + for (var i = 0; i < Threads * CookiesPerThread; i++) + { + if (provider.RemoveCookie("example.com", "/", $"k{i}") != null) + { + Interlocked.Increment(ref removed); + } + } + }).ConfigureAwait(false); + + Assert.AreEqual(Threads * CookiesPerThread, removed); + Assert.IsEmpty(provider.Cookies); + Assert.IsEmpty(new AdvancedCookieProvider(handler).Cookies); + } + + [Test] + public void CookiesCanBeEnumeratedWhileTheyChange() + { + var provider = new AdvancedCookieProvider(new MemoryFileHandler()); + ((ICookieProvider)provider).SetCookie(new Url("https://example.com/"), "a=b"); + + foreach (var cookie in provider.Cookies) + { + ((ICookieProvider)provider).SetCookie(new Url("https://example.com/"), "c=d"); + } + + Assert.AreEqual(2, provider.Cookies.Count()); + } + + private static async Task RunInParallel(Action action) + { + // own threads released together by the barrier, such that the calls really overlap + using (var barrier = new Barrier(Threads)) + { + var tasks = Enumerable.Range(0, Threads) + .Select(thread => Task.Factory.StartNew(() => + { + barrier.SignalAndWait(); + action(thread); + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)) + .ToArray(); + await Task.WhenAll(tasks).ConfigureAwait(false); + } + } + } +} diff --git a/src/AngleSharp.Io/Cookie/AdvancedCookieProvider.cs b/src/AngleSharp.Io/Cookie/AdvancedCookieProvider.cs index 0cb7ffd..65ce7a5 100644 --- a/src/AngleSharp.Io/Cookie/AdvancedCookieProvider.cs +++ b/src/AngleSharp.Io/Cookie/AdvancedCookieProvider.cs @@ -5,6 +5,7 @@ namespace AngleSharp.Io.Cookie using System; using System.Collections.Generic; using System.Linq; + using System.Threading; using static Helpers; using static NetscapeCookieSerializer; @@ -19,7 +20,11 @@ public class AdvancedCookieProvider : ICookieProvider private readonly ICookieFileHandler _handler; private readonly Boolean _forceParse; private readonly Boolean _httpOnlyExtension; - private readonly List _cookies; + // the resources of a page are loaded in parallel, so the cookies are an array that is + // never changed in place: a change swaps in a new array, and readers keep the one they got + private WebCookie[] _cookies; + private Int32 _pendingWrites; + private Int32 _writing; /// /// Creates a new cookie provider with the given handler and options. @@ -31,7 +36,7 @@ public AdvancedCookieProvider(ICookieFileHandler handler, AdvancedCookieProvider _handler = handler; _forceParse = options.IsForceParse; _httpOnlyExtension = options.IsHttpOnlyExtension; - _cookies = ReadCookies(); + _cookies = ReadCookies().ToArray(); } String ICookieProvider.GetCookie(Url url) @@ -40,8 +45,7 @@ String ICookieProvider.GetCookie(Url url) var path = String.IsNullOrEmpty(url.Path) ? "/" : $"/{url.Path}"; var secure = url.Scheme.IsOneOf(ProtocolNames.Https, ProtocolNames.Wss); var now = DateTime.UtcNow; - var cookies = _cookies - .ToArray() + var cookies = Volatile.Read(ref _cookies) .Where(c => { if (c.IsHostOnly ?? false) @@ -125,7 +129,7 @@ void ICookieProvider.SetCookie(Url url, String value) /// /// Gets the available cookies. /// - public IEnumerable Cookies => _cookies.AsEnumerable(); + public IEnumerable Cookies => EnumerateCookies(); /// /// Gets the cookie with the given key. @@ -136,8 +140,9 @@ void ICookieProvider.SetCookie(Url url, String value) /// If matching cookie, if any. public WebCookie FindCookie(String domain, String path, String key) { - domain = CanonicalDomain(domain); - return _cookies.FirstOrDefault(cookie => cookie.Domain.Is(domain) && cookie.Path.Is(path) && cookie.Key.Is(key)); + var cookies = Volatile.Read(ref _cookies); + var index = IndexOf(cookies, domain, path, key); + return index >= 0 ? cookies[index] : null; } /// @@ -149,15 +154,7 @@ public WebCookie FindCookie(String domain, String path, String key) public IEnumerable FindCookies(String domain, String path = null) { domain = CanonicalDomain(domain); - var enumerable = _cookies.Where(cookie => cookie.Domain.Is(domain)); - - if (!String.IsNullOrEmpty(path)) - { - enumerable = enumerable - .Where(cookie => CheckPaths(path, cookie.Path)); - } - - return enumerable; + return EnumerateCookies().Where(cookie => Matches(cookie, domain, path)); } /// @@ -168,8 +165,7 @@ public void AddCookie(WebCookie cookie) { if (cookie != null) { - _cookies.Remove(FindCookie(cookie.Domain, cookie.Path, cookie.Key)); - InsertCookie(cookie); + Change(cookies => Insert(Without(cookies, cookie), cookie)); WriteCookies(); } } @@ -181,12 +177,17 @@ public void AddCookie(WebCookie cookie) /// The updated cookie content. public void UpdateCookie(WebCookie oldCookie, WebCookie newCookie) { - if (oldCookie != null) + if (newCookie != null) { - _cookies.Remove(FindCookie(oldCookie.Domain, oldCookie.Path, oldCookie.Key)); + // one change, such that no reader sees the cookie missing in between + Change(cookies => Insert(Without(Without(cookies, oldCookie), newCookie), newCookie)); + WriteCookies(); + } + else if (oldCookie != null) + { + // as before, a removal without a replacement is not written to the file + Change(cookies => Without(cookies, oldCookie)); } - - AddCookie(newCookie); } /// @@ -198,11 +199,17 @@ public void UpdateCookie(WebCookie oldCookie, WebCookie newCookie) /// The removed cookie if any. public WebCookie RemoveCookie(String domain, String path, String key) { - var cookie = FindCookie(domain, path, key); + var cookie = default(WebCookie); + + Change(cookies => + { + var index = IndexOf(cookies, domain, path, key); + cookie = index >= 0 ? cookies[index] : null; + return index >= 0 ? RemoveAt(cookies, index) : cookies; + }); if (cookie != null) { - _cookies.Remove(cookie); WriteCookies(); } @@ -217,16 +224,21 @@ public WebCookie RemoveCookie(String domain, String path, String key) /// The removed cookies. public IEnumerable RemoveCookies(String domain, String path = null) { - // taken before the removal, as the lazy query would find nothing afterwards - var cookies = FindCookies(domain, path).ToArray(); + domain = CanonicalDomain(domain); + var removed = Array.Empty(); - if (cookies.Length > 0) + Change(cookies => + { + removed = cookies.Where(cookie => Matches(cookie, domain, path)).ToArray(); + return removed.Length > 0 ? cookies.Where(cookie => !Matches(cookie, domain, path)).ToArray() : cookies; + }); + + if (removed.Length > 0) { - _cookies.RemoveAll(cookie => cookies.Contains(cookie)); WriteCookies(); } - return cookies; + return removed; } /// @@ -235,33 +247,115 @@ public IEnumerable RemoveCookies(String domain, String path = null) /// The removed cookies. public IEnumerable RemoveAllCookies() { - var cookies = _cookies.ToArray(); - _cookies.RemoveAll(cookie => true); + var cookies = Interlocked.Exchange(ref _cookies, Array.Empty()); WriteCookies(); return cookies; } private List ReadCookies() => Deserialize(_handler.ReadFile(), _forceParse, _httpOnlyExtension); - private void InsertCookie(WebCookie cookie) + // reads the array when enumerated, not when called, such that Cookies stays a live view + private IEnumerable EnumerateCookies() { - for (var i = 0; i < _cookies.Count; i++) + foreach (var cookie in Volatile.Read(ref _cookies)) { - if (cookie.CompareTo(_cookies[i]) > 0) + yield return cookie; + } + } + + private void Change(Func change) + { + while (true) + { + var current = Volatile.Read(ref _cookies); + var next = change(current); + + if (ReferenceEquals(next, current) || Interlocked.CompareExchange(ref _cookies, next, current) == current) { - _cookies.Insert(i, cookie); return; } } - - _cookies.Add(cookie); } private void WriteCookies() { - var selection = _cookies.Where(m => m.IsPersistent); - var content = Serialize(selection, _httpOnlyExtension); - _handler.WriteFile(content); + Interlocked.Increment(ref _pendingWrites); + + // a single thread writes the file; a change made meanwhile only raises the counter + // and is written in that thread's next round, so nobody waits for the file + while (Interlocked.CompareExchange(ref _writing, 1, 0) == 0) + { + try + { + while (Interlocked.Exchange(ref _pendingWrites, 0) > 0) + { + var selection = Volatile.Read(ref _cookies).Where(m => m.IsPersistent); + var content = Serialize(selection, _httpOnlyExtension); + _handler.WriteFile(content); + } + } + finally + { + Volatile.Write(ref _writing, 0); + } + + // a change raised after the last round but before the release would be lost otherwise + if (Volatile.Read(ref _pendingWrites) == 0) + { + break; + } + } + } + + private static Boolean Matches(WebCookie cookie, String domain, String path) => + cookie.Domain.Is(domain) && (String.IsNullOrEmpty(path) || CheckPaths(path, cookie.Path)); + + private static Int32 IndexOf(WebCookie[] cookies, String domain, String path, String key) + { + domain = CanonicalDomain(domain); + + for (var i = 0; i < cookies.Length; i++) + { + var cookie = cookies[i]; + + if (cookie.Domain.Is(domain) && cookie.Path.Is(path) && cookie.Key.Is(key)) + { + return i; + } + } + + return -1; + } + + private static WebCookie[] Without(WebCookie[] cookies, WebCookie cookie) + { + var index = cookie != null ? IndexOf(cookies, cookie.Domain, cookie.Path, cookie.Key) : -1; + return index >= 0 ? RemoveAt(cookies, index) : cookies; + } + + private static WebCookie[] RemoveAt(WebCookie[] cookies, Int32 index) + { + var result = new WebCookie[cookies.Length - 1]; + Array.Copy(cookies, 0, result, 0, index); + Array.Copy(cookies, index + 1, result, index, result.Length - index); + return result; + } + + // keeps the order of the former list: before the first cookie it compares greater than + private static WebCookie[] Insert(WebCookie[] cookies, WebCookie cookie) + { + var index = 0; + + while (index < cookies.Length && cookie.CompareTo(cookies[index]) <= 0) + { + index++; + } + + var result = new WebCookie[cookies.Length + 1]; + Array.Copy(cookies, 0, result, 0, index); + result[index] = cookie; + Array.Copy(cookies, index, result, index + 1, cookies.Length - index); + return result; } } }