Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ Released on Sunday, November 8 2026.
- Fixed a cookie `Expires` date with a day the month does not have failing the page load @michaelthyregod
- Fixed the cookie provider failing for good once cookies were set or read from several threads at a time, as when a page loads its resources @michaelthyregod
- Fixed `RemoveCookies` returning no cookies and `RemoveAllCookies` not updating the cookie file @michaelthyregod
- Fixed parallel stores to the persistent HTTP cache failing the page load with an `IOException` @michaelthyregod
- Added trimming and Native AOT compatibility (`IsAotCompatible`) for the .NET 8 and .NET 10 targets @michaelthyregod

# 1.3.0
Expand Down
80 changes: 80 additions & 0 deletions src/AngleSharp.Io.Tests/Http/HttpCacheTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,63 @@ public async Task PersistedEntriesSurviveANewCache()
}
}

[Test]
public async Task ParallelStoresToOneOriginAllSucceed()
{
var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N"));

try
{
Func<Request, CancellationToken, Task<IResponse>> network = (request, _) =>
Task.FromResult<IResponse>(WithAddress(Ok(request.Address.Path, ("Cache-Control", "max-age=60")), request));
var cache = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) });

await StoreInParallel(64, i => cache.SendAsync(Get($"https://a.example/r{i}"), network, CancellationToken.None)).ConfigureAwait(false);

for (var i = 0; i < 64; i++)
{
var response = await cache.MatchAsync(Get($"https://a.example/r{i}"), CancellationToken.None).ConfigureAwait(false);
Assert.AreEqual($"r{i}", Read(response));
}

CollectionAssert.AreEqual(new[] { "https://a.example" }, new PersistenceHttpCacheHandler(directory).GetOrigins().ToArray());
}
finally
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, true);
}
}
}

[Test]
public async Task ParallelStoresOfOneKeyAllSucceed()
{
var directory = Path.Combine(Path.GetTempPath(), "anglesharp-http-cache-" + Guid.NewGuid().ToString("N"));

try
{
Func<Request, CancellationToken, Task<IResponse>> network = (request, _) =>
Task.FromResult<IResponse>(WithAddress(Ok("same", ("Cache-Control", "max-age=60")), request));
var cache = CreateCache(out _, new HttpCacheOptions { Handler = new PersistenceHttpCacheHandler(directory) });

await StoreInParallel(16, _ => cache.SendAsync(Get("https://a.example/page"), network, CancellationToken.None)).ConfigureAwait(false);

var response = await cache.MatchAsync(Get("https://a.example/page"), CancellationToken.None).ConfigureAwait(false);

Assert.AreEqual("same", Read(response));
Assert.IsEmpty(Directory.GetFiles(directory, "*.tmp", SearchOption.AllDirectories));
}
finally
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, true);
}
}
}

[Test]
public void WithHttpCacheWrapsHttpRequestersOnly()
{
Expand Down Expand Up @@ -601,6 +658,29 @@ private static DefaultResponse Response(HttpStatusCode status, String content, p
return response;
}

private static DefaultResponse WithAddress(DefaultResponse response, Request request)
{
response.Address = request.Address;
return response;
}

private static async Task StoreInParallel(Int32 count, Func<Int32, Task<IResponse>> send)
{
// all sends wait for the same signal, such that the stores really overlap
var start = new TaskCompletionSource<Boolean>(TaskCreationOptions.RunContinuationsAsynchronously);
var tasks = Enumerable.Range(0, count)
.Select(i => Task.Run(async () =>
{
await start.Task.ConfigureAwait(false);
using (await send(i).ConfigureAwait(false))
{
}
}))
.ToArray();
start.SetResult(true);
await Task.WhenAll(tasks).ConfigureAwait(false);
}

private static String Read(IResponse response)
{
using (var reader = new StreamReader(response.Content))
Expand Down
81 changes: 81 additions & 0 deletions src/AngleSharp.Io.Tests/WriteQueueTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
namespace AngleSharp.Io.Tests
{
using AngleSharp.Io;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;

[TestFixture]
public class WriteQueueTests
{
[Test]
public async Task WritesOfOneKeyRunOneAfterAnother()
{
var queue = new WriteQueue();
var gate = new TaskCompletionSource<Boolean>(TaskCreationOptions.RunContinuationsAsynchronously);
var order = new List<String>();

var first = queue.RunAsync("https://a.example", async () =>
{
order.Add("first started");
await gate.Task.ConfigureAwait(false);
order.Add("first done");
});
var second = queue.RunAsync("https://a.example", () =>
{
order.Add("second started");
return Task.CompletedTask;
});

// the second write may only start once the first is done, which it is not yet
Assert.IsFalse(second.IsCompleted);
CollectionAssert.AreEqual(new[] { "first started" }, order);

gate.SetResult(true);
await Task.WhenAll(first, second).ConfigureAwait(false);

CollectionAssert.AreEqual(new[] { "first started", "first done", "second started" }, order);
}

[Test]
public async Task WritesOfOtherKeysDoNotWait()
{
var queue = new WriteQueue();
var gate = new TaskCompletionSource<Boolean>(TaskCreationOptions.RunContinuationsAsynchronously);

var blocked = queue.RunAsync("https://a.example", () => gate.Task);
await queue.RunAsync("https://b.example", () => Task.CompletedTask).ConfigureAwait(false);

Assert.IsFalse(blocked.IsCompleted);
gate.SetResult(true);
await blocked.ConfigureAwait(false);
}

[Test]
public async Task AFailedWriteReportsItsErrorAndDoesNotStopTheNext()
{
var queue = new WriteQueue();
var gate = new TaskCompletionSource<Boolean>(TaskCreationOptions.RunContinuationsAsynchronously);
var ran = false;

var failing = queue.RunAsync("https://a.example", async () =>
{
await gate.Task.ConfigureAwait(false);
throw new InvalidOperationException("disk full");
});
var next = queue.RunAsync("https://a.example", () =>
{
ran = true;
return Task.CompletedTask;
});

gate.SetResult(true);

var error = Assert.ThrowsAsync<InvalidOperationException>(() => failing);
Assert.AreEqual("disk full", error.Message);
await next.ConfigureAwait(false);
Assert.IsTrue(ran);
}
}
}
13 changes: 10 additions & 3 deletions src/AngleSharp.Io/Http/HttpCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ public class HttpCache : IHttpCache

private readonly HttpCacheOptions _options;

// pages load their resources in parallel; the writes of one origin run one after
// another, such that handlers never have to deal with parallel writes
private readonly WriteQueue _writes = new WriteQueue();

#endregion

#region ctor
Expand Down Expand Up @@ -176,7 +180,7 @@ public async Task<Boolean> RemoveAsync(Url address, CancellationToken cancel)

if (variants.Count > 0)
{
await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false);
await WriteAsync(origin, key, null, cancel).ConfigureAwait(false);
return true;
}

Expand All @@ -188,7 +192,7 @@ public async Task ClearAsync(String origin, CancellationToken cancel)
{
foreach (var key in Handler.GetKeys(origin).ToArray())
{
await Handler.WriteAsync(origin, key, null, cancel).ConfigureAwait(false);
await WriteAsync(origin, key, null, cancel).ConfigureAwait(false);
}
}

Expand Down Expand Up @@ -281,9 +285,12 @@ private Task StoreAsync(String origin, String key, IReadOnlyList<HttpCacheEntry>
{
// the new entry replaces all variants the request would have selected
var remaining = variants.Where(m => !HttpCachePolicy.MatchesVary(m, request));
return Handler.WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel);
return WriteAsync(origin, key, remaining.Concat(new[] { entry }).ToArray(), cancel);
}

private Task WriteAsync(String origin, String key, IEnumerable<HttpCacheEntry> entries, CancellationToken cancel) =>
_writes.RunAsync(origin, () => Handler.WriteAsync(origin, key, entries, cancel));

private async Task InvalidateAsync(Url address, IResponse response, CancellationToken cancel)
{
var status = (Int32)response.StatusCode;
Expand Down
62 changes: 62 additions & 0 deletions src/AngleSharp.Io/WriteQueue.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
namespace AngleSharp.Io;

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading.Tasks;

/// <summary>
/// Runs the writes of one key (e.g., an origin) one after another, such that a
/// handler never sees two writes for the same key at the same time. Writes for
/// different keys still run in parallel.
/// </summary>
internal sealed class WriteQueue
{
// the last write queued per key; a key is removed again once its last write is done
private readonly ConcurrentDictionary<String, Task> _tails = new ConcurrentDictionary<String, Task>(StringComparer.Ordinal);

/// <summary>
/// Runs the given write once all writes queued before for the same key are done.
/// </summary>
/// <param name="key">The key the write belongs to.</param>
/// <param name="write">The write to run.</param>
/// <returns>The task of the write, carrying its result or failure.</returns>
public async Task RunAsync(String key, Func<Task> write)
{
key = key ?? String.Empty;
var done = new TaskCompletionSource<Boolean>(TaskCreationOptions.RunContinuationsAsynchronously);
var previous = Enqueue(key, done.Task);

try
{
// the tails always complete successfully, so a failed write does not stop the next one
await previous.ConfigureAwait(false);
await write.Invoke().ConfigureAwait(false);
}
finally
{
done.SetResult(true);

// removes the key only if no other write was queued after this one
((ICollection<KeyValuePair<String, Task>>)_tails).Remove(new KeyValuePair<String, Task>(key, done.Task));
}
}

private Task Enqueue(String key, Task tail)
{
while (true)
{
if (_tails.TryGetValue(key, out var previous))
{
if (_tails.TryUpdate(key, tail, previous))
{
return previous;
}
}
else if (_tails.TryAdd(key, tail))
{
return Task.CompletedTask;
}
}
}
}
Loading