Add custom fields for events

src/Exceptionless.Core/Exceptionless.Core.csproj

@@ -34,7 +34,7 @@
     <PackageReference Include="Stripe.net" Version="47.3.0" />
     <PackageReference Include="System.DirectoryServices" Version="9.0.1" />
     <PackageReference Include="UAParser" Version="3.1.47" />
-    <PackageReference Include="Foundatio.Repositories.Elasticsearch" Version="7.17.14" Condition="'$(ReferenceFoundatioRepositoriesSource)' == '' OR '$(ReferenceFoundatioRepositoriesSource)' == 'false'" />
+    <PackageReference Include="Foundatio.Repositories.Elasticsearch" Version="7.17.15-alpha.0.5" Condition="'$(ReferenceFoundatioRepositoriesSource)' == '' OR '$(ReferenceFoundatioRepositoriesSource)' == 'false'" />
     <ProjectReference Include="..\..\..\..\Foundatio\Foundatio.Repositories\src\Foundatio.Repositories.Elasticsearch\Foundatio.Repositories.Elasticsearch.csproj" Condition="'$(ReferenceFoundatioRepositoriesSource)' == 'true'" />
   </ItemGroup>
 

src/Exceptionless.Core/Models/PersistentEvent.cs

@@ -1,10 +1,11 @@
 using System.Diagnostics;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
 using Foundatio.Repositories.Models;
 
 namespace Exceptionless.Core.Models;
 
 [DebuggerDisplay("Id: {Id}, Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")]
-public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate
+public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate, IHaveVirtualCustomFields
 {
     /// <summary>
     /// Unique id that identifies an event.
@@ -39,5 +40,15 @@ public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWith
     /// <summary>
     /// Used to store primitive data type custom data values for searching the event.
     /// </summary>
-    public DataDictionary Idx { get; set; } = new();
+    public IDictionary<string, object?> Idx { get; set; } = new DataDictionary();
+
+    object? IHaveVirtualCustomFields.GetCustomField(string name) => Data?[name];
+    IDictionary<string, object?> IHaveVirtualCustomFields.GetCustomFields() => Data ?? [];
+    void IHaveVirtualCustomFields.RemoveCustomField(string name) => Data?.Remove(name);
+    void IHaveVirtualCustomFields.SetCustomField(string name, object value)
+    {
+        Data ??= new DataDictionary();
+        Data[name] = value;
+    }
+    string IHaveVirtualCustomFields.GetTenantKey() => OrganizationId;
 }

src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs

@@ -45,6 +45,7 @@ ILoggerFactory loggerFactory
         AddIndex(Tokens = new TokenIndex(this));
         AddIndex(Users = new UserIndex(this));
         AddIndex(WebHooks = new WebHookIndex(this));
+        AddCustomFieldIndex(_appOptions.ElasticsearchOptions.ScopePrefix + "customfields", appOptions.ElasticsearchOptions.NumberOfReplicas);
     }
 
     public Task RunAsync(CancellationToken shutdownToken = default)

src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs

@@ -21,6 +21,8 @@ public sealed class EventIndex : DailyIndex<PersistentEvent>
 
     public EventIndex(ExceptionlessElasticConfiguration configuration, IServiceProvider serviceProvider, AppOptions appOptions) : base(configuration, configuration.Options.ScopePrefix + "events", 1, doc => ((PersistentEvent)doc).Date.UtcDateTime)
     {
+        AddStandardCustomFieldTypes();
+
         _configuration = configuration;
         _serviceProvider = serviceProvider;
 
@@ -46,12 +48,6 @@ public override TypeMappingDescriptor<PersistentEvent> ConfigureIndexMapping(Typ
     {
         var mapping = map
             .Dynamic(false)
-            .DynamicTemplates(dt => dt
-                .DynamicTemplate("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => s)))
-                .DynamicTemplate("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => s)))
-                .DynamicTemplate("idx_number", t => t.Match("*-n").Mapping(m => m.Number(s => s.Type(NumberType.Double))))
-                .DynamicTemplate("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256))))
-                .DynamicTemplate("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024)))))
             .Properties(p => p
                 .SetupDefaults()
                 .Keyword(f => f.Name(e => e.Id))
@@ -74,7 +70,6 @@ public override TypeMappingDescriptor<PersistentEvent> ConfigureIndexMapping(Typ
                 .Scalar(f => f.Count)
                 .Boolean(f => f.Name(e => e.IsFirstOccurrence))
                     .FieldAlias(a => a.Name(Alias.IsFirstOccurrence).Path(f => f.IsFirstOccurrence))
-                .Object<object>(f => f.Name(e => e.Idx).Dynamic())
                 .Object<DataDictionary>(f => f.Name(e => e.Data).Properties(p2 => p2
                     .AddVersionMapping()
                     .AddLevelMapping()
@@ -147,7 +142,6 @@ protected override void ConfigureQueryParser(ElasticQueryParserConfiguration con
                 $"data.{Event.KnownDataKeys.UserInfo}.identity",
                 $"data.{Event.KnownDataKeys.UserInfo}.name"
             ])
-            .AddQueryVisitor(new EventFieldsQueryVisitor())
             .UseFieldMap(new Dictionary<string, string> {
                     { Alias.BrowserVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.BrowserVersion}" },
                     { Alias.BrowserMajorVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.BrowserMajorVersion}" },

src/Exceptionless.Core/Repositories/EventRepository.cs

@@ -1,10 +1,15 @@
-using Exceptionless.Core.Models;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
 using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Repositories.Options;
 using Exceptionless.Core.Repositories.Queries;
 using Exceptionless.DateTimeExtensions;
 using FluentValidation;
+using Foundatio.Parsers.LuceneQueries.Visitors;
 using Foundatio.Repositories;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
 using Foundatio.Repositories.Models;
+using Foundatio.Repositories.Options;
 using Nest;
 
 namespace Exceptionless.Core.Repositories;
@@ -18,11 +23,12 @@ public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptio
     {
         _timeProvider = configuration.TimeProvider;
 
+        AutoCreateCustomFields = true;
+
         DisableCache(); // NOTE: If cache is ever enabled, then fast paths for patching/deleting with scripts will be super slow!
         BatchNotifications = true;
         DefaultPipeline = "events-pipeline";
 
-        AddDefaultExclude(e => e.Idx);
         // copy to fields
         AddDefaultExclude(EventIndex.Alias.IpAddress);
         AddDefaultExclude(EventIndex.Alias.OperatingSystem);
@@ -194,4 +200,72 @@ public Task<long> RemoveAllByStackIdsAsync(string[] stackIds)
 
         return RemoveAllAsync(q => q.Stack(stackIds));
     }
+
+    protected override string? GetTenantKey(IRepositoryQuery query)
+    {
+        var organizations = query.GetOrganizations();
+        if (organizations.Count != 1)
+            return null;
+
+        return organizations.Single();
+    }
+
+    protected override async Task<CustomFieldDefinition?> HandleUnmappedCustomField(PersistentEvent document, string name, object value, IDictionary<string, CustomFieldDefinition> existingFields)
+    {
+        if (!AutoCreateCustomFields)
+            return null;
+
+        if (name.StartsWith('@'))
+            return null;
+
+        var tenantKey = GetDocumentTenantKey(document);
+        if (String.IsNullOrEmpty(tenantKey))
+            return null;
+
+        string fieldType = GetTermType(value);
+        if (fieldType == String.Empty)
+            return null;
+
+        return await ElasticIndex.Configuration.CustomFieldDefinitionRepository.AddFieldAsync(EntityTypeName, GetDocumentTenantKey(document), "data." + name, fieldType);
+    }
+
+    private static string GetTermType(object term)
+    {
+        if (term is string stringTerm)
+        {
+            if (Boolean.TryParse(stringTerm, out var _))
+                return BooleanFieldType.IndexType;
+
+            if (DateTime.TryParse(stringTerm, out var _))
+                return DateFieldType.IndexType;
+
+            return StringFieldType.IndexType;
+        }
+        else if (term is Int32)
+        {
+            return IntegerFieldType.IndexType;
+        }
+        else if (term is Int64)
+        {
+            return LongFieldType.IndexType;
+        }
+        else if (term is Double)
+        {
+            return DoubleFieldType.IndexType;
+        }
+        else if (term is float)
+        {
+            return FloatFieldType.IndexType;
+        }
+        else if (term is bool)
+        {
+            return BooleanFieldType.IndexType;
+        }
+        else if (term is DateTime or DateTimeOffset or DateOnly)
+        {
+            return DateFieldType.IndexType;
+        }
+
+        return String.Empty;
+    }
 }

tests/Exceptionless.Tests/AppWebHostFactory.cs

@@ -24,7 +24,8 @@ public Task InitializeAsync()
 
         builder.AddElasticsearch("Elasticsearch", port: 9200)
             .WithContainerName("Exceptionless-Elasticsearch-Test")
-            .WithLifetime(ContainerLifetime.Persistent);
+            .WithLifetime(ContainerLifetime.Persistent)
+            .WithKibana(b => b.WithLifetime(ContainerLifetime.Persistent).WithContainerName("Exceptionless-Kibana-Test"));
 
         _app = builder.Build();
 

tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs

@@ -11,6 +11,7 @@
 using Nest;
 using Xunit;
 using Xunit.Abstractions;
+using LogLevel = Microsoft.Extensions.Logging.LogLevel;
 
 namespace Exceptionless.Tests.Migrations;
 
@@ -39,6 +40,7 @@ protected override void RegisterServices(IServiceCollection services)
     [Fact]
     public async Task WillMergeDuplicatedStacks()
     {
+        Log.DefaultMinimumLevel = LogLevel.Trace;
         var originalStack = _stackData.GenerateStack();
         originalStack.Id = ObjectId.GenerateNewId().ToString();
         originalStack.TotalOccurrences = 100;

tests/Exceptionless.Tests/Search/EventIndexTests.cs

@@ -1,6 +1,7 @@
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Queries.Validation;
 using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Utility;
 using Exceptionless.Tests.Utility;
 using Foundatio.Repositories;
 using Foundatio.Repositories.Models;
@@ -446,6 +447,6 @@ private async Task<FindResults<PersistentEvent>> GetByFilterAsync(string filter,
         var result = await _validator.ValidateQueryAsync(filter);
         Assert.True(result.IsValid);
         Log.SetLogLevel<EventRepository>(LogLevel.Trace);
-        return await _repository.FindAsync(q => q.FilterExpression(filter));
+        return await _repository.FindAsync(q => q.Organization(SampleDataService.TEST_ORG_ID).FilterExpression(filter));
     }
 }

tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs

@@ -69,9 +69,7 @@
             return;
         }
 
-        // NOTE: we have to do this because we don't have access to the right query parser instance.
-        result = await EventFieldsQueryVisitor.RunAsync(result, context);
         Assert.Equal(expected, await GenerateQueryVisitor.RunAsync(result, context));
 
         var info = await _validator.ValidateQueryAsync(result);
         _logger.LogInformation("UsesPremiumFeatures: {UsesPremiumFeatures} IsValid: {IsValid} Message: {Message}", info.UsesPremiumFeatures, info.IsValid, info.Message);