diff --git a/docs/custom-fields.md b/docs/custom-fields.md
new file mode 100644
index 0000000000..be29887110
--- /dev/null
+++ b/docs/custom-fields.md
@@ -0,0 +1,177 @@
+# Custom Fields Architecture
+
+Custom fields let organizations index arbitrary event data properties for use in filters and search. This document covers the full lifecycle, slot system internals, deletion policy, and operator support.
+
+## Overview
+
+When an event is processed, the pipeline handler inspects the event's `Data` dictionary and writes typed values into the `Idx` sub-document using a **pooled slot** model. Rather than creating a unique Elasticsearch field per organization per field name (which would cause mapping explosion in a multi-tenant index), all organizations share a small pool of physical ES fields like `idx.keyword-1`, `idx.double-2`, etc. Each organization gets its own independent slot namespace: "department" for Org A and "department" for Org B both map to `idx.keyword-1` but are isolated by tenant-scoped queries.
+
+### Foundatio Integration
+
+The custom fields system is built on [Foundatio.Repositories.Elasticsearch custom fields](https://repositories.foundatio.dev/guide/custom-fields). Key components:
+
+- `IHaveVirtualCustomFields` — implemented by `PersistentEvent` to control how field values are read/written
+- `ICustomFieldDefinitionRepository` — stores field definitions with slot assignments per `(EntityType, TenantKey, IndexType)`
+- `EventCustomFieldService` — wires the document-changing pipeline hook and handles system field provisioning
+- `EventIndex` — registers the 8 standard custom field types via `AddStandardCustomFieldTypes()`
+
+### Supported Field Types
+
+| Type | ES Mapping | Physical Slot Pattern | Filter Operators |
+|------|-----------|----------------------|-----------------|
+| `keyword` | Keyword (exact match) | `idx.keyword-{n}` | equals, not-equals, exists, missing |
+| `string` | Text + `.keyword` sub-field | `idx.string-{n}` | contains/search, exists, missing |
+| `int` | Integer | `idx.int-{n}` | equals, gt, gte, lt, lte, range, exists, missing |
+| `long` | Long | `idx.long-{n}` | equals, gt, gte, lt, lte, range, exists, missing |
+| `double` | Double | `idx.double-{n}` | equals, gt, gte, lt, lte, range, exists, missing |
+| `float` | Float | `idx.float-{n}` | equals, gt, gte, lt, lte, range, exists, missing |
+| `bool` | Boolean | `idx.bool-{n}` | true, false, exists, missing |
+| `date` | Date | `idx.date-{n}` | equals, range, gt, gte, lt, lte, exists, missing |
+
+> **Note on `string` cost**: Each `string` slot creates **two** Elasticsearch field mappers (the `text` field and its `.keyword` sub-field), making it twice as expensive as other types toward Elasticsearch's `index.mapping.total_fields.limit` (default 1,000).
+
+## Slot System
+
+### How Slots Are Assigned
+
+Slots are assigned **sequentially** per `(EntityType, TenantKey, IndexType)` scope:
+
+```
+Org A: "department" → keyword slot 1 → idx.keyword-1
+Org A: "region" → keyword slot 2 → idx.keyword-2
+Org B: "department" → keyword slot 1 → idx.keyword-1 ← same physical field, different tenant
+Org B: "priority" → int slot 1 → idx.int-1
+```
+
+Slot assignment is protected by a **distributed lock** per scope to prevent duplicate allocation under concurrent writes.
+
+### System Fields
+
+Two system fields are provisioned automatically per organization and are **protected from deletion**:
+
+| Field Name | Type | Slot | Purpose |
+|-----------|------|------|---------|
+| `sessionend` | `date` | `date-1` | Session end timestamp (session tracking) |
+| `haserror` | `bool` | `bool-1` | Whether the session has an associated error |
+
+Because system fields are provisioned via `EnsureSystemFieldsAsync` **before** any user-defined fields, they always occupy slot 1 of their type. User fields for `date` start at slot 2; user fields for `bool` start at slot 2.
+
+If `EnsureSystemFieldsAsync` is not called (e.g., legacy org before custom fields were introduced), the first user to create a `date` or `bool` field would accidentally claim slot 1. The controller calls `EnsureSystemFieldsAsync` before every field creation to prevent this.
+
+### Slot Exhaustion and Elasticsearch Field Limits
+
+Elasticsearch's default `index.mapping.total_fields.limit` is **1,000 field mappers**. Physical slot fields are only created in the index mapping the first time a document with that slot is indexed. The maximum Elasticsearch fields from custom fields is bounded by the highest slot number ever used, multiplied by number of types, multiplied by 2 (for `string` types).
+
+With a hard limit of 20 active fields per organization and slot recycling deferred beyond the retention window, slot numbers grow slowly over time. For a typical organization cycling through fields over years, the slot high-water mark remains very low (well under 100 per type). Across many organizations sharing the same physical ES index, the absolute maximum slot number approaches the highest number ever assigned to any organization — still bounded in practice.
+
+**Field churn analysis**: The worst-case scenario is an organization that continuously creates and deletes the maximum 20 fields. Each create/delete cycle permanently consumes one slot. For any single `(EntityType, TenantKey, IndexType)` combination, the slot high-water mark is bounded by the number of distinct fields ever created. Across 8 types and 20 active fields with unlimited churn, the theoretical maximum is `8 types × unlimited cycles` — but since fields created in Elasticsearch are shared across all tenants in the index, the practical concern is the *total* unique slot number across all active organizations, not per-organization. The active per-organization cap of 20 limits how many fields a single organization can consume in any given period.
+
+There is no application-level per-type slot ceiling — the framework relies on Elasticsearch's mapping limit (default 1,000 field mappers) as the ultimate guard. A future retention-aware cleanup job will hard-delete definitions and free slots after all events indexed with those slots have aged out.
+
+## Field Lifecycle
+
+### Creating a Field
+
+1. User calls `POST /organizations/{id}/event-custom-fields`
+2. API validates: premium plan check, reserved name check, active quota check, duplicate name check
+3. `EnsureSystemFieldsAsync` provisions `sessionend` and `haserror` if not yet present
+4. `AddFieldAsync` assigns the next available slot and persists the definition
+5. **From this point on, new events with matching data keys are indexed into the slot**
+6. **Existing events are NOT backfilled** — they retain their original `Idx` content unchanged
+
+> **Search semantics on creation**: Custom field indexing applies only to events processed **after** the field definition is created. Historical events are not re-indexed. If you need historical data, you must re-ingest events or use data-level queries (`data.fieldname:value`) instead of slot queries.
+
+### Updating a Field
+
+Only `Description` and `DisplayOrder` are mutable. `Name`, `IndexType`, and `IndexSlot` are immutable once created (enforced by Foundatio's repository at save time).
+
+### Deleting a Field
+
+Deletion is a two-phase process designed to prevent **slot reuse corruption** — where a recycled slot causes historical events for a deleted field to appear in queries for a new field with the same slot.
+
+**Phase 1 — Soft Delete (synchronous):**
+1. API checks for usage in saved view filters — returns 409 Conflict if found
+2. API marks `IsDeleted = true` and calls `SaveAsync`
+3. The field name is freed from the slot system (a new field can use the same name)
+4. The slot number is **not** freed — it remains occupied
+5. New events no longer index data into this slot
+6. API returns 200 OK; the field disappears from the management UI
+7. A `RemoveCustomFieldWorkItem` is enqueued
+
+**Phase 2 — Slot Cleanup (deferred):**
+The `RemoveCustomFieldWorkItemHandler` currently **acknowledges** the soft-delete without hard-deleting the definition record. This is intentional:
+
+> **Slot Reuse Safety**: If a slot is freed and immediately recycled for a new field, historical events within the retention window that had data for the old field will appear in queries for the new field. For example: delete "customer_id" (keyword-3), create "project_id" (gets keyword-3), then searching `project_id:acme` returns historical events where `customer_id` was `acme`. This is a data integrity violation.
+
+Hard-delete (slot freeing) is deferred to a **future retention-aware cleanup job** that will only reclaim slots after all events indexed with that slot have aged out of the retention window. Until then, the slot number grows monotonically but is never reused for a different field.
+
+> **Search semantics on deletion**: After Phase 1, no new events write to the deleted slot. Existing events indexed with this field remain searchable via the slot path until they age out per the organization's retention policy. After soft-deletion and within the retention window, you may still get results from historical events if you query by slot path directly — the management UI and query builder will not surface the deleted field, so this is only visible via raw slot queries.
+
+### Slot Recycling
+
+Slot recycling (reusing a freed slot number for a new field) is **currently deferred** to prevent data contamination within the retention window. See "Deleting a Field" above. A future cleanup job will safely reclaim slots after the retention period expires.
+
+**Name reuse is always safe**: After soft-deletion, the same field *name* can be immediately reused. The new definition gets the *next available* slot number (monotonically increasing), not the old slot. This means:
+
+```
+Field A created → keyword slot 1 (active)
+Field B created → keyword slot 2 (active)
+Field A deleted → soft-delete (slot 1 still occupied, name freed)
+Field C created with same name → keyword slot 3 (new slot, no contamination)
+Query for "Field C" → only returns events since Field C was created
+```
+
+
+
+The active field limit (`MaxFieldsPerOrganization`, default 20) counts only fields that are:
+- Not soft-deleted (`IsDeleted = false`)
+- Not system fields (`sessionend`, `haserror`)
+
+Soft-deleted fields awaiting cleanup do **not** count toward the active quota. A user who has 20 active fields can delete some and immediately create replacements — the quota check reflects the current active state.
+
+### Deletion Blocked by Saved Views
+
+If a custom field is referenced in any saved view filter for the organization, deletion is blocked with HTTP 409 Conflict. The filter is checked using a regex that matches `idx.{fieldName}` tokens in the filter string. Users must remove the field from all saved view filters before deletion proceeds.
+
+## Plan Restrictions
+
+Custom fields require a paid plan. Organizations on the free plan receive HTTP 426 Upgrade Required when attempting to create a custom field. Existing fields are unaffected if an organization downgrades — they remain indexed but the management UI is read-only.
+
+## Security Model
+
+- All custom field API endpoints require authentication and verify organization ownership before any operation
+- Field names are validated against a strict allowlist (`[a-zA-Z0-9_.\-]`, max 100 chars, no `@` prefix)
+- Names starting with `@` are reserved for Exceptionless internal data keys (`@error`, `@request`, etc.)
+- Users cannot access or modify custom fields belonging to other organizations (tenant isolation enforced at the controller layer)
+- System fields (`sessionend`, `haserror`) cannot be created or deleted via the API
+
+## Elasticsearch Mapping Considerations
+
+- Custom field slot templates are registered via `AddStandardCustomFieldTypes()` in `EventIndex`
+- Templates use the pattern `idx.{type}-*` (e.g., `idx.keyword-*`, `idx.double-*`)
+- Elasticsearch creates field mappings dynamically on first document write — unused slots have zero mapping cost
+- Monitor total field count relative to `index.mapping.total_fields.limit` (default 1,000) in high-volume deployments
+- The `string` type creates 2 field mappers per slot; all other types create 1 field mapper per slot
+
+## Common Questions
+
+**Can I reuse a field name after deleting it?**
+Yes, immediately. After soft-deletion, the field name is freed and can be used for a new field. The new field gets a **new** slot number (not the old one), which prevents historical events for the deleted field from appearing in queries for the new field. Slot numbers grow monotonically and are not recycled until a future retention-aware cleanup job runs.
+
+**Does the 20-field quota include soft-deleted fields?**
+No. The quota counts only *active* (non-deleted, non-system) fields. Soft-deleted fields awaiting cleanup are excluded. You can delete fields and immediately create replacements up to the quota.
+
+**Will deleting a field break existing queries?**
+Saved view filters that reference the field are blocked at deletion time. Custom code that queries `idx.keyword-N:value` directly may stop returning results as events age out, but this is expected behavior. The Exceptionless query builder translates field names to slot paths automatically; raw slot queries are not recommended.
+
+**Is there a per-type field limit?**
+No. The active quota (`MaxFieldsPerOrganization = 20`) is a total across all types. There is no separate limit per type.
+
+**What happens if I downgrade my plan?**
+Existing field definitions and indexed data are preserved. The custom fields management UI becomes read-only. New field creation requires re-upgrading.
+
+**Can I have more than 20 fields?**
+The default limit is 20 per organization. This can be increased via the `MaxFieldsPerOrganization` configuration key for self-hosted deployments.
+
+**Can slot numbers grow unboundedly from field churn?**
+In theory, yes — each delete-then-recreate cycle adds one slot number that is never recycled. In practice, each cycle only costs one ES field mapper, and a field-churning organization (20 create/delete cycles per type) would accumulate at most ~160 slot numbers. Elasticsearch's mapping limit of 1,000 fields per index is the ultimate safety boundary. The future retention-aware cleanup job will reclaim slots and reset slot growth for organizations that heavily churn fields.
diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs
index 751d8e0880..af7b894b86 100644
--- a/src/Exceptionless.Core/Bootstrapper.cs
+++ b/src/Exceptionless.Core/Bootstrapper.cs
@@ -37,6 +37,7 @@
using Foundatio.Queues;
using Foundatio.Repositories.Elasticsearch;
using Foundatio.Repositories.Elasticsearch.Configuration;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
using Foundatio.Repositories.Elasticsearch.Jobs;
using Foundatio.Repositories.Migrations;
using Foundatio.Resilience;
@@ -93,6 +94,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
services.AddSingleton();
services.AddSingleton(s => s.GetRequiredService().Client);
services.AddSingleton(s => s.GetRequiredService());
+ services.AddSingleton(s => s.GetRequiredService().CustomFieldDefinitionRepository!);
services.AddStartupAction();
services.AddSingleton();
@@ -111,6 +113,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
+ handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
@@ -186,6 +189,8 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
services.AddSingleton();
services.AddSingleton();
services.AddSingleton();
+ services.AddSingleton();
+ services.AddStartupAction();
services.AddSingleton();
services.AddSingleton();
services.AddSingleton();
diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs
index 818b27951d..a981e7c1d1 100644
--- a/src/Exceptionless.Core/Configuration/AppOptions.cs
+++ b/src/Exceptionless.Core/Configuration/AppOptions.cs
@@ -70,6 +70,7 @@ public class AppOptions
public int BulkBatchSize { get; internal set; }
public CacheOptions CacheOptions { get; internal set; } = null!;
+ public CustomFieldOptions CustomFieldOptions { get; internal set; } = null!;
public MessageBusOptions MessageBusOptions { get; internal set; } = null!;
public QueueOptions QueueOptions { get; internal set; } = null!;
public StorageOptions StorageOptions { get; internal set; } = null!;
@@ -122,6 +123,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config)
catch { }
options.CacheOptions = CacheOptions.ReadFromConfiguration(config, options);
+ options.CustomFieldOptions = CustomFieldOptions.ReadFromConfiguration(config, options);
options.MessageBusOptions = MessageBusOptions.ReadFromConfiguration(config, options);
options.QueueOptions = QueueOptions.ReadFromConfiguration(config, options);
options.StorageOptions = StorageOptions.ReadFromConfiguration(config, options);
diff --git a/src/Exceptionless.Core/Configuration/CustomFieldOptions.cs b/src/Exceptionless.Core/Configuration/CustomFieldOptions.cs
new file mode 100644
index 0000000000..b944507919
--- /dev/null
+++ b/src/Exceptionless.Core/Configuration/CustomFieldOptions.cs
@@ -0,0 +1,16 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Exceptionless.Core.Configuration;
+
+public class CustomFieldOptions
+{
+ public int MaxFieldsPerOrganization { get; internal set; }
+
+ public static CustomFieldOptions ReadFromConfiguration(IConfiguration config, AppOptions appOptions)
+ {
+ return new CustomFieldOptions
+ {
+ MaxFieldsPerOrganization = config.GetValue(nameof(MaxFieldsPerOrganization), 20)
+ };
+ }
+}
diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj
index 2ee31516e5..b466dd3754 100644
--- a/src/Exceptionless.Core/Exceptionless.Core.csproj
+++ b/src/Exceptionless.Core/Exceptionless.Core.csproj
@@ -42,4 +42,5 @@
Include="..\..\..\..\Foundatio\Foundatio.Repositories\src\Foundatio.Repositories.Elasticsearch\Foundatio.Repositories.Elasticsearch.csproj"
Condition="'$(ReferenceFoundatioRepositoriesSource)' == 'true'" />
+
diff --git a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs
index c0901a893f..f92f16be06 100644
--- a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs
+++ b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs
@@ -9,77 +9,6 @@ public static class PersistentEventExtensions
{
private static readonly char[] _commaSeparator = [','];
- public static void CopyDataToIndex(this PersistentEvent ev, string[]? keysToCopy = null)
- {
- if (ev.Data is null)
- return;
-
- ev.Idx ??= new DataDictionary();
-
- keysToCopy = keysToCopy?.Length > 0 ? keysToCopy : ev.Data.Keys.ToArray();
-
- foreach (string key in keysToCopy.Where(k => !String.IsNullOrEmpty(k) && ev.Data.ContainsKey(k)))
- {
- string field = key.Trim().ToLowerInvariant();
-
- if (field.StartsWith("@ref:"))
- {
- field = field.Substring(5);
- if (!field.IsValidFieldName())
- continue;
-
- ev.Idx[field + "-r"] = ev.Data[key]?.ToString();
- continue;
- }
-
- if (field.StartsWith('@') || ev.Data[key] is null)
- continue;
-
- if (!field.IsValidFieldName())
- continue;
-
- var dataType = ev.Data[key]?.GetType();
- if (dataType is null)
- continue;
-
- if (dataType == typeof(bool))
- {
- ev.Idx[field + "-b"] = ev.Data[key];
- }
- else if (dataType.IsNumeric())
- {
- ev.Idx[field + "-n"] = ev.Data[key];
- }
- else if (dataType == typeof(DateTime) || dataType == typeof(DateTimeOffset))
- {
- ev.Idx[field + "-d"] = ev.Data[key];
- }
- else if (dataType == typeof(string))
- {
- string? input = ev.Data[key]?.ToString();
- if (String.IsNullOrEmpty(input) || input.Length >= 1000)
- continue;
-
- if (input.GetJsonType() != JsonType.None)
- continue;
-
- if (input[0] == '"')
- input = input.TrimStart('"').TrimEnd('"');
-
- if (Boolean.TryParse(input, out bool value))
- ev.Idx[field + "-b"] = value;
- else if (DateTimeOffset.TryParse(input, out var dtoValue))
- ev.Idx[field + "-d"] = dtoValue;
- else if (Decimal.TryParse(input, out decimal decValue))
- ev.Idx[field + "-n"] = decValue;
- else if (Double.TryParse(input, out double dblValue) && !Double.IsNaN(dblValue) && !Double.IsInfinity(dblValue))
- ev.Idx[field + "-n"] = dblValue;
- else
- ev.Idx[field + "-s"] = input;
- }
- }
- }
-
public static string? GetEventReference(this PersistentEvent ev, string name)
{
if (String.IsNullOrEmpty(name) || ev.Data is null)
@@ -146,7 +75,7 @@ public static bool HasSessionEndTime(this PersistentEvent ev)
return null;
}
- public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActivityUtc, bool isSessionEnd = false)
+ public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false)
{
if (!ev.IsSessionStart())
return false;
@@ -167,18 +96,21 @@ public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActi
if (isSessionEnd)
{
ev.Data[Event.KnownDataKeys.SessionEnd] = lastActivityUtc;
- ev.CopyDataToIndex([Event.KnownDataKeys.SessionEnd]);
}
else
{
ev.Data.Remove(Event.KnownDataKeys.SessionEnd);
- ev.Idx?.Remove(Event.KnownDataKeys.SessionEnd + "-d");
}
+ if (hasError)
+ ev.Data[Event.KnownDataKeys.SessionHasError] = true;
+ else
+ ev.Data.Remove(Event.KnownDataKeys.SessionHasError);
+
return true;
}
- public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, JsonSerializerOptions jsonOptions, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool hasPremiumFeatures = true, bool includePrivateInformation = true)
+ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, JsonSerializerOptions jsonOptions, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool includePrivateInformation = true)
{
var startEvent = new PersistentEvent
{
@@ -239,9 +171,6 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J
if (lastActivityUtc.HasValue)
startEvent.UpdateSessionStart(lastActivityUtc.Value, isSessionEnd.GetValueOrDefault());
- if (hasPremiumFeatures)
- startEvent.CopyDataToIndex([]);
-
return startEvent;
}
diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs
index 73e886505c..625e13f0db 100644
--- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs
+++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs
@@ -2,6 +2,7 @@
using Exceptionless.Core.Models;
using Exceptionless.Core.Models.WorkItems;
using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services;
using Foundatio.Jobs;
using Foundatio.Lock;
using Foundatio.Repositories;
@@ -13,13 +14,15 @@ public class OrganizationMaintenanceWorkItemHandler : WorkItemHandlerBase
{
private readonly IOrganizationRepository _organizationRepository;
private readonly BillingManager _billingManager;
+ private readonly EventCustomFieldService _eventCustomFieldService;
private readonly TimeProvider _timeProvider;
private readonly ILockProvider _lockProvider;
- public OrganizationMaintenanceWorkItemHandler(IOrganizationRepository organizationRepository, ILockProvider lockProvider, BillingManager billingManager, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+ public OrganizationMaintenanceWorkItemHandler(IOrganizationRepository organizationRepository, ILockProvider lockProvider, BillingManager billingManager, EventCustomFieldService eventCustomFieldService, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
{
_organizationRepository = organizationRepository;
_billingManager = billingManager;
+ _eventCustomFieldService = eventCustomFieldService;
_timeProvider = timeProvider;
_lockProvider = lockProvider;
}
@@ -34,7 +37,8 @@ public override async Task HandleItemAsync(WorkItemContext context)
const int LIMIT = 100;
var wi = context.GetData()!;
- Log.LogInformation("Received upgrade organizations work item. Upgrade Plans: {UpgradePlans}", wi.UpgradePlans);
+ Log.LogInformation("Received organization maintenance work item. UpgradePlans: {UpgradePlans} RemoveOldUsageStats: {RemoveOldUsageStats} EnsureSystemCustomFields: {EnsureSystemCustomFields}",
+ wi.UpgradePlans, wi.RemoveOldUsageStats, wi.EnsureSystemCustomFields);
var results = await _organizationRepository.GetAllAsync(o => o.PageLimit(LIMIT));
while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
@@ -53,6 +57,18 @@ public override async Task HandleItemAsync(WorkItemContext context)
foreach (var usage in organization.Usage.Where(u => u.Date < utcNow.Subtract(TimeSpan.FromDays(366))).ToList())
organization.Usage.Remove(usage);
}
+
+ if (wi.EnsureSystemCustomFields)
+ {
+ try
+ {
+ await _eventCustomFieldService.EnsureSystemFieldsAsync(organization.Id);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ Log.LogError(ex, "Error ensuring system custom fields for organization {OrganizationId}", organization.Id);
+ }
+ }
}
if (wi.UpgradePlans || wi.RemoveOldUsageStats)
diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveCustomFieldWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveCustomFieldWorkItemHandler.cs
new file mode 100644
index 0000000000..a054d274dc
--- /dev/null
+++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveCustomFieldWorkItemHandler.cs
@@ -0,0 +1,76 @@
+using Exceptionless.Core.Models.WorkItems;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+public class RemoveCustomFieldWorkItemHandler : WorkItemHandlerBase
+{
+ private readonly ICustomFieldDefinitionRepository _customFieldDefinitionRepository;
+ private readonly ILockProvider _lockProvider;
+
+ public RemoveCustomFieldWorkItemHandler(
+ ICustomFieldDefinitionRepository customFieldDefinitionRepository,
+ ILockProvider lockProvider,
+ ILoggerFactory loggerFactory) : base(loggerFactory)
+ {
+ _customFieldDefinitionRepository = customFieldDefinitionRepository;
+ _lockProvider = lockProvider;
+ }
+
+ public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
+ {
+ string cacheKey = $"{nameof(RemoveCustomFieldWorkItem)}:{((RemoveCustomFieldWorkItem)workItem).CustomFieldDefinitionId}";
+ return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken);
+ }
+
+ public override async Task HandleItemAsync(WorkItemContext context)
+ {
+ var workItem = context.GetData()!;
+
+ using (Log.BeginScope(new ExceptionlessState().Organization(workItem.OrganizationId)))
+ {
+ Log.LogInformation("Processing custom field removal work item for definition: {DefinitionId}, field: {FieldName}",
+ workItem.CustomFieldDefinitionId, workItem.FieldName);
+
+ await context.ReportProgressAsync(0, "Acknowledging custom field soft-deletion...");
+
+ // GetByIdAsync INCLUDES soft-deleted records (by-ID lookups bypass the soft-delete filter).
+ // After the controller soft-deletes and enqueues this work item, GetByIdAsync returns the
+ // definition with IsDeleted=true. A null result means the record has been physically removed.
+ var definition = await _customFieldDefinitionRepository.GetByIdAsync(workItem.CustomFieldDefinitionId);
+
+ if (definition is null)
+ {
+ Log.LogWarning(
+ "Custom field definition {DefinitionId} ('{FieldName}') no longer exists. " +
+ "It may have been hard-deleted externally.",
+ workItem.CustomFieldDefinitionId, workItem.FieldName);
+ }
+ else if (!definition.IsDeleted)
+ {
+ Log.LogWarning(
+ "Custom field definition {DefinitionId} ('{FieldName}') is unexpectedly active. " +
+ "It should have been soft-deleted before this work item was enqueued.",
+ workItem.CustomFieldDefinitionId, workItem.FieldName);
+ }
+ else
+ {
+ // Normal path: definition is soft-deleted as expected.
+ // Hard-deletion (slot reclamation) is intentionally deferred to prevent slot-reuse corruption:
+ // recycling a slot before the org's retention window expires would cause historical events
+ // indexed under the old field to appear in queries for a new field assigned the same slot.
+ // A future retention-aware cleanup job will hard-delete once all events using the old slot
+ // have aged out beyond the org's retention period.
+ Log.LogInformation(
+ "Custom field definition {DefinitionId} ('{FieldName}') is soft-deleted. " +
+ "Slot will be reclaimed by the retention cleanup job after the retention window expires.",
+ workItem.CustomFieldDefinitionId, workItem.FieldName);
+ }
+
+ await context.ReportProgressAsync(100, $"Custom field '{workItem.FieldName}' acknowledged.");
+ }
+ }
+}
diff --git a/src/Exceptionless.Core/Models/PersistentEvent.cs b/src/Exceptionless.Core/Models/PersistentEvent.cs
index 968a4acd37..a1e3426bff 100644
--- a/src/Exceptionless.Core/Models/PersistentEvent.cs
+++ b/src/Exceptionless.Core/Models/PersistentEvent.cs
@@ -2,12 +2,13 @@
using System.Diagnostics;
using Exceptionless.Core.Attributes;
using Exceptionless.Core.Extensions;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
using Foundatio.Repositories.Models;
namespace Exceptionless.Core.Models;
[DebuggerDisplay("Id: {Id}, Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")]
-public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate, IValidatableObject
+public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate, IValidatableObject, IHaveVirtualCustomFields
{
///
/// Unique id that identifies an event.
@@ -52,6 +53,28 @@ public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWith
[MiniValidation.SkipRecursion]
public DataDictionary? Idx { get; set; }
+ // IHaveVirtualCustomFields explicit implementation
+ IDictionary IHaveVirtualCustomFields.Idx => (IDictionary)(Idx ??= new DataDictionary());
+
+ public string GetTenantKey() => OrganizationId;
+
+ public IDictionary GetCustomFields()
+ {
+ if (Data is null) return new DataDictionary();
+ var result = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ foreach (var kvp in Data.Where(kvp => !String.IsNullOrEmpty(kvp.Key)
+ && (!kvp.Key.StartsWith('@') || kvp.Key.StartsWith("@ref:", StringComparison.OrdinalIgnoreCase))
+ && kvp.Value is string or bool or int or long or float or double or decimal or DateTime or DateTimeOffset))
+ {
+ result[kvp.Key] = kvp.Value;
+ }
+ return result;
+ }
+
+ public object GetCustomField(string name) => Data is not null && Data.TryGetValue(name, out var v) && v is not null ? v : null!;
+ public void SetCustomField(string name, object value) { Data ??= new DataDictionary(); Data[name] = value; }
+ public void RemoveCustomField(string name) => Data?.Remove(name);
+
public IEnumerable Validate(ValidationContext validationContext)
{
if (Date == DateTimeOffset.MinValue)
diff --git a/src/Exceptionless.Core/Models/SavedView.cs b/src/Exceptionless.Core/Models/SavedView.cs
index 83e83ed0f8..cea1476f8f 100644
--- a/src/Exceptionless.Core/Models/SavedView.cs
+++ b/src/Exceptionless.Core/Models/SavedView.cs
@@ -81,6 +81,9 @@ public record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates
/// Schema version for future filter definition migrations.
public int Version { get; set; } = 1;
+ /// True when the filter references at least one custom field or other premium feature.
+ public bool UsesPremiumFeatures { get; set; }
+
/// Dashboard page identifier: "events", "stacks", or "stream".
[Required]
[RegularExpression("^(events|stacks|stream)$")]
diff --git a/src/Exceptionless.Core/Models/WorkItems/OrganizationMaintenanceWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/OrganizationMaintenanceWorkItem.cs
index d619d1a3dd..2bd1d04ca6 100644
--- a/src/Exceptionless.Core/Models/WorkItems/OrganizationMaintenanceWorkItem.cs
+++ b/src/Exceptionless.Core/Models/WorkItems/OrganizationMaintenanceWorkItem.cs
@@ -4,4 +4,5 @@ public class OrganizationMaintenanceWorkItem
{
public bool UpgradePlans { get; set; }
public bool RemoveOldUsageStats { get; set; }
+ public bool EnsureSystemCustomFields { get; set; }
}
diff --git a/src/Exceptionless.Core/Models/WorkItems/RemoveCustomFieldWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/RemoveCustomFieldWorkItem.cs
new file mode 100644
index 0000000000..25485a1e07
--- /dev/null
+++ b/src/Exceptionless.Core/Models/WorkItems/RemoveCustomFieldWorkItem.cs
@@ -0,0 +1,8 @@
+namespace Exceptionless.Core.Models.WorkItems;
+
+public record RemoveCustomFieldWorkItem
+{
+ public required string OrganizationId { get; init; }
+ public required string CustomFieldDefinitionId { get; init; }
+ public required string FieldName { get; init; }
+}
diff --git a/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs b/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs
deleted file mode 100644
index b85723efa2..0000000000
--- a/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs
+++ /dev/null
@@ -1,29 +0,0 @@
-using Exceptionless.Core.Plugins.EventProcessor;
-using Microsoft.Extensions.Logging;
-
-namespace Exceptionless.Core.Pipeline;
-
-[Priority(40)]
-public class CopySimpleDataToIdxAction : EventPipelineActionBase
-{
- public CopySimpleDataToIdxAction(AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { }
-
- public override Task ProcessAsync(EventContext ctx)
- {
- if (!ctx.Organization.HasPremiumFeatures)
- return Task.CompletedTask;
-
- // TODO: Do we need a pipeline action to trim keys and remove null values that may be sent by other native clients.
- ctx.Event.CopyDataToIndex([]);
- int fieldCount = ctx.Event.Idx?.Count ?? 0;
- AppDiagnostics.EventsFieldCount.Record(fieldCount);
- if (fieldCount > 20 && _logger.IsEnabled(LogLevel.Warning))
- {
- var ev = ctx.Event;
- using (_logger.BeginScope(new ExceptionlessState().Organization(ctx.Organization.Id).Property("Event", new { ev.Date, ev.StackId, ev.Type, ev.Source, ev.Message, ev.Value, ev.Geo, ev.ReferenceId, ev.Tags, ev.Idx })))
- _logger.LogWarning("Event has {FieldCount} indexed fields", fieldCount);
- }
-
- return Task.CompletedTask;
- }
-}
diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs
index 42b0169be7..972ca7c2b9 100644
--- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs
+++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs
@@ -89,7 +89,8 @@ private async Task ProcessManualSessionsAsync(ICollection contexts
});
// try to update an existing session
- string? sessionStartEventId = await UpdateSessionStartEventAsync(projectId, session.Key, lastSessionEvent.Event.Date.UtcDateTime, sessionEndEvent is not null);
+ bool sessionHasError = session.Any(ctx => String.Equals(ctx.Event.Type, Event.KnownTypes.Error, StringComparison.Ordinal));
+ string? sessionStartEventId = await UpdateSessionStartEventAsync(projectId, session.Key, lastSessionEvent.Event.Date.UtcDateTime, sessionEndEvent is not null, sessionHasError);
// do we already have a session start for this session id?
if (!String.IsNullOrEmpty(sessionStartEventId) && sessionStartEvent is not null)
@@ -179,11 +180,12 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts)
session.ForEach(s => s.Event.SetSessionId(sessionId));
+ bool identitySessionHasError = session.Any(ctx => String.Equals(ctx.Event.Type, Event.KnownTypes.Error, StringComparison.Ordinal));
if (isNewSession)
{
if (sessionStartEvent is not null)
{
- sessionStartEvent.Event.UpdateSessionStart(lastSessionEvent.Event.Date.UtcDateTime, lastSessionEvent.Event.IsSessionEnd());
+ sessionStartEvent.Event.UpdateSessionStart(lastSessionEvent.Event.Date.UtcDateTime, lastSessionEvent.Event.IsSessionEnd(), identitySessionHasError);
sessionStartEvent.SetProperty("SetSessionStartEventId", true);
}
else
@@ -203,7 +205,7 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts)
sessionStartEvent.IsCancelled = true;
}
- await UpdateSessionStartEventAsync(projectId, sessionId, lastSessionEvent.Event.Date.UtcDateTime, lastSessionEvent.Event.IsSessionEnd());
+ await UpdateSessionStartEventAsync(projectId, sessionId, lastSessionEvent.Event.Date.UtcDateTime, lastSessionEvent.Event.IsSessionEnd(), identitySessionHasError);
}
}
}
@@ -286,7 +288,7 @@ private Task SetIdentitySessionIdAsync(string projectId, string identity,
private async Task CreateSessionStartEventAsync(EventContext startContext, DateTime? lastActivityUtc, bool? isSessionEnd)
{
- var startEvent = startContext.Event.ToSessionStartEvent(_jsonOptions, lastActivityUtc, isSessionEnd, startContext.Organization.HasPremiumFeatures, startContext.IncludePrivateInformation);
+ var startEvent = startContext.Event.ToSessionStartEvent(_jsonOptions, lastActivityUtc, isSessionEnd, startContext.IncludePrivateInformation);
var startEventContexts = new List {
new(startEvent, startContext.Organization, startContext.Project)
};
diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs
index 2158adeade..a1fdfd2f0d 100644
--- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs
+++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs
@@ -10,6 +10,7 @@
using Foundatio.Queues;
using Foundatio.Repositories.Elasticsearch;
using Foundatio.Repositories.Elasticsearch.Configuration;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
using Foundatio.Repositories.Elasticsearch.Queries.Builders;
using Foundatio.Resilience;
using Microsoft.Extensions.Logging;
@@ -41,6 +42,7 @@ ILoggerFactory loggerFactory
_logger.LogInformation("All new indexes will be created with {ElasticsearchNumberOfShards} Shards and {ElasticsearchNumberOfReplicas} Replicas", _appOptions.ElasticsearchOptions.NumberOfShards, _appOptions.ElasticsearchOptions.NumberOfReplicas);
AddIndex(Stacks = new StackIndex(this));
AddIndex(Events = new EventIndex(this, serviceProvider, appOptions));
+ AddCustomFieldIndex(_appOptions.ElasticsearchOptions.ScopePrefix + "customfields", appOptions.ElasticsearchOptions.NumberOfReplicas);
AddIndex(Migrations = new MigrationIndex(this, _appOptions.ElasticsearchOptions.ScopePrefix + "migrations", appOptions.ElasticsearchOptions.NumberOfReplicas));
AddIndex(Organizations = new OrganizationIndex(this));
AddIndex(Projects = new ProjectIndex(this));
diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs
index 268b6ddc6f..c783e33002 100644
--- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs
+++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs
@@ -21,6 +21,8 @@ public sealed class EventIndex : DailyIndex
public EventIndex(ExceptionlessElasticConfiguration configuration, IServiceProvider serviceProvider, AppOptions appOptions) : base(configuration, configuration.Options.ScopePrefix + "events", 1, doc => ((PersistentEvent)doc).Date.UtcDateTime)
{
+ AddStandardCustomFieldTypes();
+
_configuration = configuration;
_serviceProvider = serviceProvider;
@@ -46,12 +48,6 @@ public override TypeMappingDescriptor ConfigureIndexMapping(Typ
{
var mapping = map
.Dynamic(false)
- .DynamicTemplates(dt => dt
- .DynamicTemplate("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => s)))
- .DynamicTemplate("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => s)))
- .DynamicTemplate("idx_number", t => t.Match("*-n").Mapping(m => m.Number(s => s.Type(NumberType.Double))))
- .DynamicTemplate("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256))))
- .DynamicTemplate("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024)))))
.Properties(p => p
.SetupDefaults()
.Keyword(f => f.Name(e => e.Id))
@@ -74,7 +70,6 @@ public override TypeMappingDescriptor ConfigureIndexMapping(Typ
.Scalar(f => f.Count)
.Boolean(f => f.Name(e => e.IsFirstOccurrence))
.FieldAlias(a => a.Name(Alias.IsFirstOccurrence).Path(f => f.IsFirstOccurrence))
- .Object