From af9be042f9fbe8b12889d98b0e03191abdeff294 Mon Sep 17 00:00:00 2001 From: Michael Thyregod <2386496+michaelthyregod@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:42:45 +0100 Subject: [PATCH 1/3] Let parallel stores to the persistent HTTP cache succeed Every store rewrote the origin.txt of its origin, so parallel loads from one origin collided on it (51 of 64 parallel stores failed in a repro), and stores of one key could collide on the swap. The IOException reached the page load. origin.txt is only written when missing, as its content follows from the folder name. A store that still collides with another one is dropped and its temporary file deleted: the other entry is as good, and like a failed read, a failed store only means a cache miss later. --- CHANGELOG.md | 4 + .../Http/HttpCacheTests.cs | 80 ++++++++++++++++++ .../Http/PersistenceHttpCacheHandler.cs | 81 +++++++++++++++---- 3 files changed, 149 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d899d1b..5057887 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 1.3.2 + +- Fixed parallel stores to the persistent HTTP cache failing the page load with an `IOException` @michaelthyregod + # 1.3.1 Released on Sunday, November 8 2026. diff --git a/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs b/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs index dc74a2e..342047b 100644 --- a/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs +++ b/src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs @@ -462,6 +462,63 @@ public async Task PersistedEntriesSurviveANewCache() } } + [Test] + public async Task ParallelStoresToOneOriginAllSucceed() + { + var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N")); + + try + { + Func> network = (request, _) => + Task.FromResult(WithAddress(Ok(request.Address.Path, ("Cache-Control", "max-age=60")), request)); + var cache = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) }); + + await StoreInParallel(64, i => cache.SendAsync(Get($"https://a.example/r{i}"), network, CancellationToken.None)).ConfigureAwait(false); + + for (var i = 0; i < 64; i++) + { + var response = await cache.MatchAsync(Get($"https://a.example/r{i}"), CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual($"r{i}", Read(response)); + } + + CollectionAssert.AreEqual(new[] { "https://a.example" }, new PersistenceHttpCacheHandler(directory).GetOrigins().ToArray()); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + } + } + + [Test] + public async Task ParallelStoresOfOneKeyAllSucceed() + { + var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N")); + + try + { + Func> network = (request, _) => + Task.FromResult(WithAddress(Ok("same", ("Cache-Control", "max-age=60")), request)); + var cache = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) }); + + await StoreInParallel(16, _ => cache.SendAsync(Get("https://a.example/page"), network, CancellationToken.None)).ConfigureAwait(false); + + var response = await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None).ConfigureAwait(false); + + Assert.AreEqual("same", Read(response)); + Assert.IsEmpty(Directory.GetFiles(directory, "*.tmp", SearchOption.AllDirectories)); + } + finally + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, true); + } + } + } + [Test] public void WithHttpCacheWrapsHttpRequestersOnly() { @@ -601,6 +658,29 @@ private static DefaultResponse Response(HttpStatusCode status, String content, p return response; } + private static DefaultResponse WithAddress(DefaultResponse response, Request request) + { + response.Address = request.Address; + return response; + } + + private static async Task StoreInParallel(Int32 count, Func> send) + { + // all sends wait for the same signal, such that the stores really overlap + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tasks = Enumerable.Range(0, count) + .Select(i => Task.Run(async () => + { + await start.Task.ConfigureAwait(false); + using (await send(i).ConfigureAwait(false)) + { + } + })) + .ToArray(); + start.SetResult(true); + await Task.WhenAll(tasks).ConfigureAwait(false); + } + private static String Read(IResponse response) { using (var reader = new StreamReader(response.Content)) diff --git a/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs b/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs index 754e354..89ae4d0 100644 --- a/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs +++ b/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs @@ -66,39 +66,56 @@ public async Task WriteAsync(String origin, String key, IEnumerable GetKeys(String origin) return result; } + private static void WriteOriginFile(String originFile, String origin) + { + // the folder is named by the hash of the origin, so an existing file already has this + // content; writing it again on every store made parallel stores collide on it + if (!File.Exists(originFile)) + { + try + { + using (var stream = new FileStream(originFile, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) + using (var writer = new StreamWriter(stream)) + { + writer.Write(origin ?? String.Empty); + } + } + catch (IOException) when (File.Exists(originFile)) + { + // created by another store at the same moment + } + } + } + + private static void TryDelete(String filePath) + { + try + { + File.Delete(filePath); + } + catch (IOException) + { + } + } + private String GetOriginPath(String origin) => Path.Combine(_directoryPath, Hash(origin ?? String.Empty)); From eef7ead5b9383a2842f242357132db9110825acb Mon Sep 17 00:00:00 2001 From: Michael Thyregod <2386496+michaelthyregod@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:45:15 +0100 Subject: [PATCH 2/3] Move the CHANGELOG line into the unreleased 1.3.1 --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5057887..84ec1c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,10 @@ -# 1.3.2 - -- Fixed parallel stores to the persistent HTTP cache failing the page load with an `IOException` @michaelthyregod - # 1.3.1 Released on Sunday, November 8 2026. - Fixed `GetAttachedFileName` passing on folders from `Content-Disposition`; it now gives a plain file name @michaelthyregod - Fixed a cookie `Expires` date with a day the month does not have failing the page load @michaelthyregod +- Fixed parallel stores to the persistent HTTP cache failing the page load with an `IOException` @michaelthyregod - Added trimming and Native AOT compatibility (`IsAotCompatible`) for the .NET 8 and .NET 10 targets @michaelthyregod # 1.3.0 From 37231cead602c7f4c92f9807ae779dd7d322027e Mon Sep 17 00:00:00 2001 From: Michael Thyregod <2386496+michaelthyregod@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:18:34 +0100 Subject: [PATCH 3/3] Run the HTTP cache writes of one origin one after another HttpCache now queues its handler writes per origin (WriteQueue), so handlers never see parallel writes; the changes to PersistenceHttpCacheHandler are undone. --- src/AngleSharp.Io.Tests/WriteQueueTests.cs | 81 +++++++++++++++++++ src/AngleSharp.Io/Http/HttpCache.cs | 13 ++- .../Http/PersistenceHttpCacheHandler.cs | 81 ++++--------------- src/AngleSharp.Io/WriteQueue.cs | 62 ++++++++++++++ 4 files changed, 169 insertions(+), 68 deletions(-) create mode 100644 src/AngleSharp.Io.Tests/WriteQueueTests.cs create mode 100644 src/AngleSharp.Io/WriteQueue.cs diff --git a/src/AngleSharp.Io.Tests/WriteQueueTests.cs b/src/AngleSharp.Io.Tests/WriteQueueTests.cs new file mode 100644 index 0000000..11a9895 --- /dev/null +++ b/src/AngleSharp.Io.Tests/WriteQueueTests.cs @@ -0,0 +1,81 @@ +namespace AngleSharp.Io.Tests +{ + using AngleSharp.Io; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + + [TestFixture] + public class WriteQueueTests + { + [Test] + public async Task WritesOfOneKeyRunOneAfterAnother() + { + var queue = new WriteQueue(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var order = new List(); + + var first = queue.RunAsync("https://a.example", async () => + { + order.Add("first started"); + await gate.Task.ConfigureAwait(false); + order.Add("first done"); + }); + var second = queue.RunAsync("https://a.example", () => + { + order.Add("second started"); + return Task.CompletedTask; + }); + + // the second write may only start once the first is done, which it is not yet + Assert.IsFalse(second.IsCompleted); + CollectionAssert.AreEqual(new[] { "first started" }, order); + + gate.SetResult(true); + await Task.WhenAll(first, second).ConfigureAwait(false); + + CollectionAssert.AreEqual(new[] { "first started", "first done", "second started" }, order); + } + + [Test] + public async Task WritesOfOtherKeysDoNotWait() + { + var queue = new WriteQueue(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var blocked = queue.RunAsync("https://a.example", () => gate.Task); + await queue.RunAsync("https://b.example", () => Task.CompletedTask).ConfigureAwait(false); + + Assert.IsFalse(blocked.IsCompleted); + gate.SetResult(true); + await blocked.ConfigureAwait(false); + } + + [Test] + public async Task AFailedWriteReportsItsErrorAndDoesNotStopTheNext() + { + var queue = new WriteQueue(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ran = false; + + var failing = queue.RunAsync("https://a.example", async () => + { + await gate.Task.ConfigureAwait(false); + throw new InvalidOperationException("disk full"); + }); + var next = queue.RunAsync("https://a.example", () => + { + ran = true; + return Task.CompletedTask; + }); + + gate.SetResult(true); + + var error = Assert.ThrowsAsync(() => failing); + Assert.AreEqual("disk full", error.Message); + await next.ConfigureAwait(false); + Assert.IsTrue(ran); + } + } +} diff --git a/src/AngleSharp.Io/Http/HttpCache.cs b/src/AngleSharp.Io/Http/HttpCache.cs index 1923afb..0e09640 100644 --- a/src/AngleSharp.Io/Http/HttpCache.cs +++ b/src/AngleSharp.Io/Http/HttpCache.cs @@ -20,6 +20,10 @@ public class HttpCache : IHttpCache private readonly HttpCacheOptions _options; + // pages load their resources in parallel; the writes of one origin run one after + // another, such that handlers never have to deal with parallel writes + private readonly WriteQueue _writes = new WriteQueue(); + #endregion #region ctor @@ -176,7 +180,7 @@ public async Task RemoveAsync(Url address, CancellationToken cancel) if (variants.Count > 0) { - await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false); + await WriteAsync(origin, key, null, cancel).ConfigureAwait(false); return true; } @@ -188,7 +192,7 @@ public async Task ClearAsync(String origin, CancellationToken cancel) { foreach (var key in Handler.GetKeys(origin).ToArray()) { - await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false); + await WriteAsync(origin, key, null, cancel).ConfigureAwait(false); } } @@ -281,9 +285,12 @@ private Task StoreAsync(String origin, String key, IReadOnlyList { // the new entry replaces all variants the request would have selected var remaining = variants.Where(m => !HttpCachePolicy.MatchesVary(m, request)); - return Handler.WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel); + return WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel); } + private Task WriteAsync(String origin, String key, IEnumerable entries, CancellationToken cancel) => + _writes.RunAsync(origin, () => Handler.WriteAsync(origin, key, entries, cancel)); + private async Task InvalidateAsync(Url address, IResponse response, CancellationToken cancel) { var status = (Int32)response.StatusCode; diff --git a/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs b/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs index 89ae4d0..754e354 100644 --- a/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs +++ b/src/AngleSharp.Io/Http/PersistenceHttpCacheHandler.cs @@ -66,56 +66,39 @@ public async Task WriteAsync(String origin, String key, IEnumerable GetKeys(String origin) return result; } - private static void WriteOriginFile(String originFile, String origin) - { - // the folder is named by the hash of the origin, so an existing file already has this - // content; writing it again on every store made parallel stores collide on it - if (!File.Exists(originFile)) - { - try - { - using (var stream = new FileStream(originFile, FileMode.CreateNew, FileAccess.Write, FileShare.Read)) - using (var writer = new StreamWriter(stream)) - { - writer.Write(origin ?? String.Empty); - } - } - catch (IOException) when (File.Exists(originFile)) - { - // created by another store at the same moment - } - } - } - - private static void TryDelete(String filePath) - { - try - { - File.Delete(filePath); - } - catch (IOException) - { - } - } - private String GetOriginPath(String origin) => Path.Combine(_directoryPath, Hash(origin ?? String.Empty)); diff --git a/src/AngleSharp.Io/WriteQueue.cs b/src/AngleSharp.Io/WriteQueue.cs new file mode 100644 index 0000000..e009dc2 --- /dev/null +++ b/src/AngleSharp.Io/WriteQueue.cs @@ -0,0 +1,62 @@ +namespace AngleSharp.Io; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; + +/// +/// Runs the writes of one key (e.g., an origin) one after another, such that a +/// handler never sees two writes for the same key at the same time. Writes for +/// different keys still run in parallel. +/// +internal sealed class WriteQueue +{ + // the last write queued per key; a key is removed again once its last write is done + private readonly ConcurrentDictionary _tails = new ConcurrentDictionary(StringComparer.Ordinal); + + /// + /// Runs the given write once all writes queued before for the same key are done. + /// + /// The key the write belongs to. + /// The write to run. + /// The task of the write, carrying its result or failure. + public async Task RunAsync(String key, Func write) + { + key = key ?? String.Empty; + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var previous = Enqueue(key, done.Task); + + try + { + // the tails always complete successfully, so a failed write does not stop the next one + await previous.ConfigureAwait(false); + await write.Invoke().ConfigureAwait(false); + } + finally + { + done.SetResult(true); + + // removes the key only if no other write was queued after this one + ((ICollection>)_tails).Remove(new KeyValuePair(key, done.Task)); + } + } + + private Task Enqueue(String key, Task tail) + { + while (true) + { + if (_tails.TryGetValue(key, out var previous)) + { + if (_tails.TryUpdate(key, tail, previous)) + { + return previous; + } + } + else if (_tails.TryAdd(key, tail)) + { + return Task.CompletedTask; + } + } + } +}