diff --git a/CHANGELOG.md b/CHANGELOG.md index d2d2fdb..701fc5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Released on Sunday, November 8 2026. - Fixed a cookie `Expires` date with a day the month does not have failing the page load @michaelthyregod - Fixed the cookie provider failing for good once cookies were set or read from several threads at a time, as when a page loads its resources @michaelthyregod - Fixed `RemoveCookies` returning no cookies and `RemoveAllCookies` not updating the cookie file @michaelthyregod +- Fixed parallel stores to the persistent HTTP cache failing the page load with an `IOException` @michaelthyregod - Added trimming and Native AOT compatibility (`IsAotCompatible`) for the .NET 8 and .NET 10 targets @michaelthyregod # 1.3.0 diff --git a/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs b/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs index dc74a2e..342047b 100644 --- a/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs +++ b/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs @@ -462,6 +462,63 @@ public async Task PersistedEntriesSurviveANewCache() } } + [Test] + public async Task ParallelStoresToOneOriginAllSucceed() + { + var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N")); + + try + { + Func> network = (request, _) => + Task.FromResult(WithAddress(Ok(request.Address.Path, ("Cache-Control", "max-age=60")), request)); + var cache = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) }); + + await StoreInParallel(64, i => cache.SendAsync(Get($"https://a.example/r{i}"), network, CancellationToken.None)).ConfigureAwait(false); + + for (var i = 0; i < 64; i++) + { + var response = await cache.MatchAsync(Get($"https://a.example/r{i}"), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual($"r{i}", Read(response)); + } + + CollectionAssert.AreEqual(new[] { "https://a.example" }, new PersistenceHttpCacheHandler(directory).GetOrigins().ToArray()); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + } + } + + [Test] + public async Task ParallelStoresOfOneKeyAllSucceed() + { + var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N")); + + try + { + Func> network = (request, _) => + Task.FromResult(WithAddress(Ok("same", ("Cache-Control", "max-age=60")), request)); + var cache = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) }); + + await StoreInParallel(16, _ => cache.SendAsync(Get("https://a.example/page"), network, CancellationToken.None)).ConfigureAwait(false); + + var response = await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual("same", Read(response)); + Assert.IsEmpty(Directory.GetFiles(directory, "*.tmp", SearchOption.AllDirectories)); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + } + } + [Test] public void WithHttpCacheWrapsHttpRequestersOnly() { @@ -601,6 +658,29 @@ private static DefaultResponse Response(HttpStatusCode status, String content, p return response; } + private static DefaultResponse WithAddress(DefaultResponse response, Request request) + { + response.Address = request.Address; + return response; + } + + private static async Task StoreInParallel(Int32 count, Func> send) + { + // all sends wait for the same signal, such that the stores really overlap + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tasks = Enumerable.Range(0, count) + .Select(i => Task.Run(async () => + { + await start.Task.ConfigureAwait(false); + using (await send(i).ConfigureAwait(false)) + { + } + })) + .ToArray(); + start.SetResult(true); + await Task.WhenAll(tasks).ConfigureAwait(false); + } + private static String Read(IResponse response) { using (var reader = new StreamReader(response.Content)) diff --git a/src/AngleSharp.Io.Tests/WriteQueueTests.cs b/src/AngleSharp.Io.Tests/WriteQueueTests.cs new file mode 100644 index 0000000..11a9895 --- /dev/null +++ b/src/AngleSharp.Io.Tests/WriteQueueTests.cs @@ -0,0 +1,81 @@ +namespace AngleSharp.Io.Tests +{ + using AngleSharp.Io; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + + [TestFixture] + public class WriteQueueTests + { + [Test] + public async Task WritesOfOneKeyRunOneAfterAnother() + { + var queue = new WriteQueue(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var order = new List(); + + var first = queue.RunAsync("https://a.example", async () => + { + order.Add("first started"); + await gate.Task.ConfigureAwait(false); + order.Add("first done"); + }); + var second = queue.RunAsync("https://a.example", () => + { + order.Add("second started"); + return Task.CompletedTask; + }); + + // the second write may only start once the first is done, which it is not yet + Assert.IsFalse(second.IsCompleted); + CollectionAssert.AreEqual(new[] { "first started" }, order); + + gate.SetResult(true); + await Task.WhenAll(first, second).ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { "first started", "first done", "second started" }, order); + } + + [Test] + public async Task WritesOfOtherKeysDoNotWait() + { + var queue = new WriteQueue(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var blocked = queue.RunAsync("https://a.example", () => gate.Task); + await queue.RunAsync("https://b.example", () => Task.CompletedTask).ConfigureAwait(false); + + Assert.IsFalse(blocked.IsCompleted); + gate.SetResult(true); + await blocked.ConfigureAwait(false); + } + + [Test] + public async Task AFailedWriteReportsItsErrorAndDoesNotStopTheNext() + { + var queue = new WriteQueue(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ran = false; + + var failing = queue.RunAsync("https://a.example", async () => + { + await gate.Task.ConfigureAwait(false); + throw new InvalidOperationException("disk full"); + }); + var next = queue.RunAsync("https://a.example", () => + { + ran = true; + return Task.CompletedTask; + }); + + gate.SetResult(true); + + var error = Assert.ThrowsAsync(() => failing); + Assert.AreEqual("disk full", error.Message); + await next.ConfigureAwait(false); + Assert.IsTrue(ran); + } + } +} diff --git a/src/AngleSharp.Io/Http/HttpCache.cs b/src/AngleSharp.Io/Http/HttpCache.cs index 1923afb..0e09640 100644 --- a/src/AngleSharp.Io/Http/HttpCache.cs +++ b/src/AngleSharp.Io/Http/HttpCache.cs @@ -20,6 +20,10 @@ public class HttpCache : IHttpCache private readonly HttpCacheOptions _options; + // pages load their resources in parallel; the writes of one origin run one after + // another, such that handlers never have to deal with parallel writes + private readonly WriteQueue _writes = new WriteQueue(); + #endregion #region ctor @@ -176,7 +180,7 @@ public async Task RemoveAsync(Url address, CancellationToken cancel) if (variants.Count > 0) { - await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false); + await WriteAsync(origin, key, null, cancel).ConfigureAwait(false); return true; } @@ -188,7 +192,7 @@ public async Task ClearAsync(String origin, CancellationToken cancel) { foreach (var key in Handler.GetKeys(origin).ToArray()) { - await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false); + await WriteAsync(origin, key, null, cancel).ConfigureAwait(false); } } @@ -281,9 +285,12 @@ private Task StoreAsync(String origin, String key, IReadOnlyList { // the new entry replaces all variants the request would have selected var remaining = variants.Where(m => !HttpCachePolicy.MatchesVary(m, request)); - return Handler.WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel); + return WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel); } + private Task WriteAsync(String origin, String key, IEnumerable entries, CancellationToken cancel) => + _writes.RunAsync(origin, () => Handler.WriteAsync(origin, key, entries, cancel)); + private async Task InvalidateAsync(Url address, IResponse response, CancellationToken cancel) { var status = (Int32)response.StatusCode; diff --git a/src/AngleSharp.Io/WriteQueue.cs b/src/AngleSharp.Io/WriteQueue.cs new file mode 100644 index 0000000..e009dc2 --- /dev/null +++ b/src/AngleSharp.Io/WriteQueue.cs @@ -0,0 +1,62 @@ +namespace AngleSharp.Io; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; + +/// +/// Runs the writes of one key (e.g., an origin) one after another, such that a +/// handler never sees two writes for the same key at the same time. Writes for +/// different keys still run in parallel. +/// +internal sealed class WriteQueue +{ + // the last write queued per key; a key is removed again once its last write is done + private readonly ConcurrentDictionary _tails = new ConcurrentDictionary(StringComparer.Ordinal); + + /// + /// Runs the given write once all writes queued before for the same key are done. + /// + /// The key the write belongs to. + /// The write to run. + /// The task of the write, carrying its result or failure. + public async Task RunAsync(String key, Func write) + { + key = key ?? String.Empty; + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var previous = Enqueue(key, done.Task); + + try + { + // the tails always complete successfully, so a failed write does not stop the next one + await previous.ConfigureAwait(false); + await write.Invoke().ConfigureAwait(false); + } + finally + { + done.SetResult(true); + + // removes the key only if no other write was queued after this one + ((ICollection>)_tails).Remove(new KeyValuePair(key, done.Task)); + } + } + + private Task Enqueue(String key, Task tail) + { + while (true) + { + if (_tails.TryGetValue(key, out var previous)) + { + if (_tails.TryUpdate(key, tail, previous)) + { + return previous; + } + } + else if (_tails.TryAdd(key, tail)) + { + return Task.CompletedTask; + } + } + } +}