In-flight items pinned to an older baseline image are dispatched with a model ID their pinned agent CLI cannot serve

docs/reference/api.md

@@ -1668,11 +1668,38 @@ Response: `200 OK`
 ```
 
 Inactive when the registered sandbox provider does not implement baseline
-imaging (process / bubblewrap) — the response is then `{ "baselines": [], "sweepEntries": 0 }`.
+imaging (process / bubblewrap) — the response is then `{ "baselines": [],
+"sweepEntries": 0, "pinConsistency": { …all-zero/null… } }`.
+
+Both baseline endpoints also return a `pinConsistency` object: the
+advisory check for pins whose baked baseline differs from the live config's
+(e.g. an agent CLI upgrade moved the baseline under in-flight items while
+their dispatch model also changed — a combination the baked CLI may not
+know). `warning` is `null` when nothing is stale; otherwise it names the
+count, the pickup-time auto-migration policy mode, and points at
+`POST /baselines/migrate`. `GET /baselines` embeds the per-item `stalePins`
+list; the dashboard summary reports counts only:
+
+```json
+{
+  "pinConsistency": {
+    "scanned": 5,
+    "staleToolchain": 2,
+    "truncated": false,
+    "warning": "2 pinned work item(s) are stale: … run POST /baselines/migrate to move them now.",
+    "stalePins": [
+      { "id": "0199…", "projectId": "p", "pinnedRef": "incus/tc-aaaa…/vm-old", "pinnedModel": "old-model", "liveRef": "incus/tc-bbbb…/vm-new" }
+    ]
+  }
+}
+```
 
 ### `GET /admin/baseline-images`
 
 Compact dashboard summary of the latest sweep. Counts only; no work-item IDs.
+Includes the `pinConsistency` counts and warning (no per-item list) so
+dashboards can alert when a config change leaves in-flight items on an
+inconsistent CLI/model combination.
 
 Response: `200 OK`
 
@@ -1684,7 +1711,8 @@ Response: `200 OK`
   "entries": [
     { "name": "cb-baseline-9c74f712aa85", "isLive": true, "ageInGraceMinutes": null },
     { "name": "cb-baseline-bc5b1542b6d6", "isLive": false, "ageInGraceMinutes": 240.0 }
-  ]
+  ],
+  "pinConsistency": { "scanned": 5, "staleToolchain": 0, "truncated": false, "warning": null, "stalePins": [] }
 }
 ```
 

src/CodeyBox.Api/BaselineEndpoints.cs

@@ -101,7 +101,11 @@ private static BaselineMigrationFilter ParseFilter(MigrateBaselineRequest? reque
     }
 
     /// <summary>
-    /// Returns the latest baseline image sweep with referencing work items.
+    /// Returns the latest baseline image sweep with referencing work items,
+    /// plus the pin-consistency check: pins whose baked toolchain differs from
+    /// the live config's (a config change moved the baseline under in-flight
+    /// items). <c>pinConsistency.warning</c> is null when nothing is stale;
+    /// otherwise it names the count and points at <c>POST /baselines/migrate</c>.
     /// Returned shape:
     /// <code>
     /// {
@@ -114,13 +118,19 @@ private static BaselineMigrationFilter ParseFilter(MigrateBaselineRequest? reque
     ///       "workItems": [{"id":"…","title":"…","state":3}]
     ///     }
     ///   ],
-    ///   "sweepEntries": 12
+    ///   "sweepEntries": 12,
+    ///   "pinConsistency": {
+    ///     "scanned": 5, "staleToolchain": 2, "truncated": false,
+    ///     "warning": "…",
+    ///     "stalePins": [{"id":"…","projectId":"…","pinnedRef":"…","pinnedModel":"…","liveRef":"…"}]
+    ///   }
     /// }
     /// </code>
     /// </summary>
     private static async Task<IResult> GetBaselinesAsync(
         BaselineImageReaper reaper,
         IWorkItemStore store,
+        BaselineMigrationService migrator,
         CancellationToken ct)
     {
         var report = reaper.GetLatestReport();
@@ -148,14 +158,21 @@ private static async Task<IResult> GetBaselinesAsync(
         {
             baselines = entries,
             sweepEntries = report.Count,
+            pinConsistency = await BuildPinConsistencyAsync(migrator, includeItems: true, ct),
         });
     }
 
     /// <summary>
     /// Compact summary intended for fleet dashboards. Counts only — no work-
-    /// item identifiers — so it stays cheap to render frequently.
+    /// item identifiers — so it stays cheap to render frequently. Includes the
+    /// stale-toolchain pin count and warning so dashboards can alert when a
+    /// config change leaves in-flight items on an inconsistent CLI/model
+    /// combination.
     /// </summary>
-    private static IResult GetBaselineSummary(BaselineImageReaper reaper)
+    private static async Task<IResult> GetBaselineSummary(
+        BaselineImageReaper reaper,
+        BaselineMigrationService migrator,
+        CancellationToken ct)
     {
         var report = reaper.GetLatestReport();
         return Results.Ok(new
@@ -171,6 +188,30 @@ private static IResult GetBaselineSummary(BaselineImageReaper reaper)
                     ? (double?)Math.Round(m, 1)
                     : null,
             }).ToArray(),
+            pinConsistency = await BuildPinConsistencyAsync(migrator, includeItems: false, ct),
         });
     }
+
+    private static async Task<object> BuildPinConsistencyAsync(
+        BaselineMigrationService migrator, bool includeItems, CancellationToken ct)
+    {
+        var consistency = await migrator.CheckConsistencyAsync(ct);
+        return new
+        {
+            scanned = consistency.ScannedCount,
+            staleToolchain = consistency.StaleToolchainCount,
+            truncated = consistency.Truncated,
+            warning = consistency.Warning,
+            stalePins = includeItems
+                ? consistency.StalePins.Select(p => new
+                {
+                    id = p.Id.ToString(),
+                    projectId = p.ProjectId.Value,
+                    pinnedRef = p.PinnedRef,
+                    pinnedModel = p.PinnedModelId,
+                    liveRef = p.LiveRef,
+                }).ToArray()
+                : Array.Empty<object>(),
+        };
+    }
 }

src/CodeyBox.Api/Program.cs

@@ -4542,7 +4542,8 @@ static Func<TestRunOptions> DotnetTestRunOptionsAccessor(IServiceProvider sp)
     burnEstimatorOptions: sp.GetService<AgentBurnEstimatorOptions>(),
     dispatchLiveness: sp.GetRequiredService<DispatchItemLivenessTracker>(),
     hostTransientPause: sp.GetRequiredService<ProviderTransientCorrelationTracker>(),
-    baseBrokenConditions: sp.GetRequiredService<BaseBrokenConditionTracker>()));
+    baseBrokenConditions: sp.GetRequiredService<BaseBrokenConditionTracker>(),
+    stalePinOptions: () => sp.GetRequiredService<IOptionsMonitor<CodeyBoxOptions>>().CurrentValue.BaselineStalePin));
 builder.Services.AddSingleton<IInfrastructureDeferralScheduler>(
     sp => sp.GetRequiredService<OrchestratorService>());
 builder.Services.AddSingleton<IRefactorProjectGateStatusProvider>(
@@ -4857,7 +4858,8 @@ static Func<TestRunOptions> DotnetTestRunOptionsAccessor(IServiceProvider sp)
         sp.GetRequiredService<IProjectRepository>(),
         sp.GetService<TimeProvider>() ?? TimeProvider.System,
         () => monitor.CurrentValue.BaselineMigration,
-        sp.GetRequiredService<ILogger<BaselineMigrationService>>());
+        sp.GetRequiredService<ILogger<BaselineMigrationService>>(),
+        () => monitor.CurrentValue.BaselineStalePin);
 });
 
 // --- Stale-base PR sweeper ---------------------------------------------------
@@ -6958,6 +6960,17 @@ public RepoRetentionOptions RepoReaper
         /// </summary>
         public BaselineMigrationOptions BaselineMigration { get; set; } = new();
 
+        /// <summary>
+        /// Pickup-time stale-baseline-pin policy. When a work item's pinned
+        /// baseline was baked from different agent executable/toolchain inputs
+        /// than the live config resolves now, and the dispatch model has also
+        /// changed since the pin, the pickup clears the pin so the item
+        /// recomputes onto the current baseline (same outcome as a manual
+        /// migrate) instead of running a model the baked CLI may not know.
+        /// See <see cref="BaselineStalePinEvaluator"/>.
+        /// </summary>
+        public BaselineStalePinOptions BaselineStalePin { get; set; } = new();
+
         /// <summary>
         /// Stale-base PR sweeper configuration. Detects open CodeyBox-authored
         /// PRs whose base branch has moved and produced a merge conflict the

src/CodeyBox.Core/AgentFailureClassifier.cs

@@ -341,6 +341,13 @@ public static AgentFailureClassification Classify(
             return new AgentFailureClassification(AgentFailureKind.Infrastructure, Reason: "agent binary was not found in the sandbox");
         }
 
+        if (FindUnknownModelEvidence(stderr, stdout, summary) is { } unknownModel)
+        {
+            return new AgentFailureClassification(
+                AgentFailureKind.Infrastructure,
+                Reason: $"agent CLI does not know the requested model (pattern '{unknownModel.Pattern}' on {unknownModel.Stream}: '{unknownModel.Line}')");
+        }
+
         var noCapturedOutput = string.IsNullOrEmpty(stderr) && string.IsNullOrEmpty(stdout);
 
         if (ContainsAny(stderr, HardQuotaPatterns) || ContainsAny(stdout, HardQuotaPatterns))
@@ -704,6 +711,201 @@ private static (string Pattern, string Line)? FindAuthErrorEvidence(string? stde
         return null;
     }
 
+    /// <summary>
+    /// CLI-reported "the requested model is not known to this CLI" shapes.
+    /// Deliberately CLI-voiced ("unknown model", "model not found") rather
+    /// than bare "model": model prose ("the model failed to compile") must
+    /// never classify as a configuration fault. Matched case-insensitively
+    /// across stderr, stdout, and summary so an agy-style glog line folded
+    /// into any of them still classifies. Real CLI errors name the rejected
+    /// id inline (<c>model 'x' not found</c>), so a second anchored check
+    /// below catches those with the id quoted between the words.
+    /// </summary>
+    public static readonly IReadOnlyList<string> UnknownModelPatterns = new[]
+    {
+        "unknown model",
+        "unsupported model",
+        "unrecognized model",
+        "unrecognised model",
+        "invalid model",
+        "model not found",
+        "model was not found",
+        "no such model",
+        "model does not exist",
+        "unknown --model",
+        "invalid --model",
+        "not a valid model",
+    };
+
+    /// <summary>
+    /// Error words that, on the same line as an anchored model reference
+    /// (<c>model 'id'</c>, <c>model "id"</c>, <c>model: id</c>,
+    /// <c>model=id</c>, <c>--model</c>), mark a CLI model refusal even when
+    /// the quoted id sits between the words (<c>model 'x' not found</c>).
+    /// </summary>
+    private static readonly IReadOnlyList<string> UnknownModelErrorWords = new[]
+    {
+        "unknown",
+        "invalid",
+        "unsupported",
+        "unrecognized",
+        "unrecognised",
+        "not found",
+        "no such",
+        "does not exist",
+        "doesn't exist",
+    };
+
+    private static readonly IReadOnlyList<string> UnknownModelAnchors = new[]
+    {
+        "model '",
+        "model \"",
+        "model:",
+        "model=",
+        "--model",
+    };
+
+    /// <summary>
+    /// True when any captured stream carries a CLI-reported unknown-model
+    /// shape. Used by the audit path to fail fast with a deterministic
+    /// configuration error instead of burning the fresh-sandbox retry on a
+    /// command that will fail identically. Never throws.
+    /// </summary>
+    public static bool IsUnknownModelFailure(string? stderr, string? stdout, string? summary = null) =>
+        FindUnknownModelEvidence(stderr, stdout, summary) is not null;
+
+    /// <summary>
+    /// Finds the first unknown-model line across stderr, stdout, then summary
+    /// (that priority order), returning the matched pattern, the stream, and
+    /// the truncated line for reasons. Returns null when nothing matches.
+    /// </summary>
+    public static (string Pattern, string Stream, string Line)? FindUnknownModelEvidence(
+        string? stderr, string? stdout, string? summary = null) =>
+        FindInStream(stderr, "stderr")
+        ?? FindInStream(stdout, "stdout")
+        ?? FindInStream(summary, "summary");
+
+    private static (string Pattern, string Stream, string Line)? FindInStream(string? text, string stream)
+    {
+        if (string.IsNullOrEmpty(text))
+            return null;
+        foreach (var rawLine in text.Split('\n'))
+        {
+            var line = rawLine.Trim();
+            if (string.IsNullOrEmpty(line))
+                continue;
+            foreach (var pattern in UnknownModelPatterns)
+            {
+                if (line.Contains(pattern, StringComparison.OrdinalIgnoreCase))
+                    return (pattern, stream, TruncateEvidenceLine(line));
+            }
+            if (FindAnchoredModelRefusal(line) is { } word)
+                return (word, stream, TruncateEvidenceLine(line));
+        }
+
+        return null;
+    }
+
+    /// <summary>
+    /// Catches <c>model 'id' not found</c>-shaped refusals the fixed patterns
+    /// miss because the quoted id breaks the word adjacency. Requires BOTH an
+    /// anchored model reference and an error word on the same line, so bare
+    /// prose ("the model failed", "model output was empty") never matches.
+    /// The error word must sit OUTSIDE the quoted model id as a standalone
+    /// token: in <c>model 'unknown-xyz' is not served by the provider
+    /// endpoint</c> the word "unknown" is part of the id, not a CLI refusal,
+    /// so that provider-side failure must not classify as unknown-model.
+    /// Returns the matched error word for reasons, else null.
+    /// </summary>
+    private static string? FindAnchoredModelRefusal(string line)
+    {
+        if (!ContainsAny(line, UnknownModelAnchors))
+            return null;
+        var unquoted = StripQuotedSegments(line);
+        for (var i = 0; i < UnknownModelErrorWords.Count; i++)
+        {
+            if (ContainsTokenSequence(unquoted, UnknownModelErrorWords[i]))
+                return UnknownModelErrorWords[i];
+        }
+        return null;
+    }
+
+    /// <summary>
+    /// Blanks out paired single/double-quoted segments (typically the rejected
+    /// model id) so <see cref="FindAnchoredModelRefusal"/> cannot match an
+    /// error word that only occurs inside the id. An unmatched quote (for
+    /// example the apostrophe in "doesn't") leaves the rest of the line
+    /// untouched.
+    /// </summary>
+    private static string StripQuotedSegments(string line)
+    {
+        var chars = line.ToCharArray();
+        var i = 0;
+        while (i < chars.Length)
+        {
+            if (!IsQuoteChar(chars[i]))
+            {
+                i++;
+                continue;
+            }
+            var closer = IndexOfQuoteChar(chars, i + 1);
+            if (closer < 0)
+                break;
+            for (var j = i; j <= closer; j++)
+                chars[j] = ' ';
+            i = closer + 1;
+        }
+        return new string(chars);
+    }
+
+    private static bool IsQuoteChar(char ch) =>
+        ch == '\'' || ch == '"' || ch == '\u2018' || ch == '\u2019' || ch == '\u201C' || ch == '\u201D';
+
+    private static int IndexOfQuoteChar(char[] chars, int start)
+    {
+        for (var i = start; i < chars.Length; i++)
+        {
+            if (IsQuoteChar(chars[i]))
+                return i;
+        }
+        return -1;
+    }
+
+    /// <summary>
+    /// Word-boundary punctuation stripped from token edges before matching.
+    /// Dashes, underscores, slashes, and dots are deliberately NOT stripped:
+    /// they join model-id parts (<c>unknown-xyz</c>), so trimming them would
+    /// reintroduce the false positive this guard exists to prevent.
+    /// </summary>
+    private static readonly char[] TokenEdgePunctuation =
+        ['.', ',', ';', ':', '!', '?', '(', ')', '[', ']', '{', '}', '\'', '"', '\u2018', '\u2019', '\u201C', '\u201D', '`', '*'];
+
+    /// <summary>
+    /// Case-insensitive whole-token (or adjacent-token for multi-word phrases)
+    /// match, so <c>unknown-xyz</c> never matches the error word
+    /// <c>unknown</c> while <c>not found.</c> still matches <c>not found</c>.
+    /// </summary>
+    private static bool ContainsTokenSequence(string text, string word)
+    {
+        var tokens = text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
+        var parts = word.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+        for (var i = 0; i + parts.Length <= tokens.Length; i++)
+        {
+            var match = true;
+            for (var j = 0; j < parts.Length; j++)
+            {
+                if (!tokens[i + j].Trim(TokenEdgePunctuation).Equals(parts[j], StringComparison.OrdinalIgnoreCase))
+                {
+                    match = false;
+                    break;
+                }
+            }
+            if (match)
+                return true;
+        }
+        return false;
+    }
+
     /// <summary>
     /// Guarded stdout matcher. Delegates to the same shared predicate as
     /// <see cref="ContainsAuthRequiredPatternInStderr"/> so the two streams