diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index df96d642c2..3411632ac7 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -34,7 +34,7 @@ - + diff --git a/src/Exceptionless.Core/Models/PersistentEvent.cs b/src/Exceptionless.Core/Models/PersistentEvent.cs index 229127796b..cf0c834d25 100644 --- a/src/Exceptionless.Core/Models/PersistentEvent.cs +++ b/src/Exceptionless.Core/Models/PersistentEvent.cs @@ -1,10 +1,11 @@ using System.Diagnostics; +using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Models; namespace Exceptionless.Core.Models; [DebuggerDisplay("Id: {Id}, Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")] -public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate +public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate, IHaveVirtualCustomFields { /// /// Unique id that identifies an event. @@ -39,5 +40,15 @@ public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWith /// /// Used to store primitive data type custom data values for searching the event. /// - public DataDictionary Idx { get; set; } = new(); + public IDictionary Idx { get; set; } = new DataDictionary(); + + object? IHaveVirtualCustomFields.GetCustomField(string name) => Data?[name]; + IDictionary IHaveVirtualCustomFields.GetCustomFields() => Data ?? []; + void IHaveVirtualCustomFields.RemoveCustomField(string name) => Data?.Remove(name); + void IHaveVirtualCustomFields.SetCustomField(string name, object value) + { + Data ??= new DataDictionary(); + Data[name] = value; + } + string IHaveVirtualCustomFields.GetTenantKey() => OrganizationId; } diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index e8eaa7e37e..a23ac9fa12 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -45,6 +45,7 @@ ILoggerFactory loggerFactory AddIndex(Tokens = new TokenIndex(this)); AddIndex(Users = new UserIndex(this)); AddIndex(WebHooks = new WebHookIndex(this)); + AddCustomFieldIndex(_appOptions.ElasticsearchOptions.ScopePrefix + "customfields", appOptions.ElasticsearchOptions.NumberOfReplicas); } public Task RunAsync(CancellationToken shutdownToken = default) diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index 725bd7672b..eab8d9ed0b 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -21,6 +21,8 @@ public sealed class EventIndex : DailyIndex public EventIndex(ExceptionlessElasticConfiguration configuration, IServiceProvider serviceProvider, AppOptions appOptions) : base(configuration, configuration.Options.ScopePrefix + "events", 1, doc => ((PersistentEvent)doc).Date.UtcDateTime) { + AddStandardCustomFieldTypes(); + _configuration = configuration; _serviceProvider = serviceProvider; @@ -46,12 +48,6 @@ public override TypeMappingDescriptor ConfigureIndexMapping(Typ { var mapping = map .Dynamic(false) - .DynamicTemplates(dt => dt - .DynamicTemplate("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => s))) - .DynamicTemplate("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => s))) - .DynamicTemplate("idx_number", t => t.Match("*-n").Mapping(m => m.Number(s => s.Type(NumberType.Double)))) - .DynamicTemplate("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256)))) - .DynamicTemplate("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024))))) .Properties(p => p .SetupDefaults() .Keyword(f => f.Name(e => e.Id)) @@ -74,7 +70,6 @@ public override TypeMappingDescriptor ConfigureIndexMapping(Typ .Scalar(f => f.Count) .Boolean(f => f.Name(e => e.IsFirstOccurrence)) .FieldAlias(a => a.Name(Alias.IsFirstOccurrence).Path(f => f.IsFirstOccurrence)) - .Object(f => f.Name(e => e.Idx).Dynamic()) .Object(f => f.Name(e => e.Data).Properties(p2 => p2 .AddVersionMapping() .AddLevelMapping() @@ -147,7 +142,6 @@ protected override void ConfigureQueryParser(ElasticQueryParserConfiguration con $"data.{Event.KnownDataKeys.UserInfo}.identity", $"data.{Event.KnownDataKeys.UserInfo}.name" ]) - .AddQueryVisitor(new EventFieldsQueryVisitor()) .UseFieldMap(new Dictionary { { Alias.BrowserVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.BrowserVersion}" }, { Alias.BrowserMajorVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.BrowserMajorVersion}" }, diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index 65e1d7ad43..c2206301d4 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -1,10 +1,15 @@ -using Exceptionless.Core.Models; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Options; using Exceptionless.Core.Repositories.Queries; using Exceptionless.DateTimeExtensions; using FluentValidation; +using Foundatio.Parsers.LuceneQueries.Visitors; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.CustomFields; using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; using Nest; namespace Exceptionless.Core.Repositories; @@ -18,11 +23,12 @@ public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptio { _timeProvider = configuration.TimeProvider; + AutoCreateCustomFields = true; + DisableCache(); // NOTE: If cache is ever enabled, then fast paths for patching/deleting with scripts will be super slow! BatchNotifications = true; DefaultPipeline = "events-pipeline"; - AddDefaultExclude(e => e.Idx); // copy to fields AddDefaultExclude(EventIndex.Alias.IpAddress); AddDefaultExclude(EventIndex.Alias.OperatingSystem); @@ -194,4 +200,72 @@ public Task RemoveAllByStackIdsAsync(string[] stackIds) return RemoveAllAsync(q => q.Stack(stackIds)); } + + protected override string? GetTenantKey(IRepositoryQuery query) + { + var organizations = query.GetOrganizations(); + if (organizations.Count != 1) + return null; + + return organizations.Single(); + } + + protected override async Task HandleUnmappedCustomField(PersistentEvent document, string name, object value, IDictionary existingFields) + { + if (!AutoCreateCustomFields) + return null; + + if (name.StartsWith('@')) + return null; + + var tenantKey = GetDocumentTenantKey(document); + if (String.IsNullOrEmpty(tenantKey)) + return null; + + string fieldType = GetTermType(value); + if (fieldType == String.Empty) + return null; + + return await ElasticIndex.Configuration.CustomFieldDefinitionRepository.AddFieldAsync(EntityTypeName, GetDocumentTenantKey(document), "data." + name, fieldType); + } + + private static string GetTermType(object term) + { + if (term is string stringTerm) + { + if (Boolean.TryParse(stringTerm, out var _)) + return BooleanFieldType.IndexType; + + if (DateTime.TryParse(stringTerm, out var _)) + return DateFieldType.IndexType; + + return StringFieldType.IndexType; + } + else if (term is Int32) + { + return IntegerFieldType.IndexType; + } + else if (term is Int64) + { + return LongFieldType.IndexType; + } + else if (term is Double) + { + return DoubleFieldType.IndexType; + } + else if (term is float) + { + return FloatFieldType.IndexType; + } + else if (term is bool) + { + return BooleanFieldType.IndexType; + } + else if (term is DateTime or DateTimeOffset or DateOnly) + { + return DateFieldType.IndexType; + } + + return String.Empty; + } } diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs deleted file mode 100644 index 0c4e7d6104..0000000000 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs +++ /dev/null @@ -1,131 +0,0 @@ -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Models; -using Foundatio.Parsers.LuceneQueries.Nodes; -using Foundatio.Parsers.LuceneQueries.Visitors; - -namespace Exceptionless.Core.Repositories.Queries; - -public class EventFieldsQueryVisitor : ChainableQueryVisitor -{ - public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context) - { - var childTerms = new List(); - if (node.Left is TermNode { Field: null } leftTermNode) - childTerms.Add(leftTermNode.Term); - - if (node.Left is TermRangeNode { Field: null } leftTermRangeNode) - { - childTerms.Add(leftTermRangeNode.Min); - childTerms.Add(leftTermRangeNode.Max); - } - - if (node.Right is TermNode { Field: null } rightTermNode) - childTerms.Add(rightTermNode.Term); - - if (node.Right is TermRangeNode { Field: null } rightTermRangeNode) - { - childTerms.Add(rightTermRangeNode.Min); - childTerms.Add(rightTermRangeNode.Max); - } - - node.Field = GetCustomFieldName(node.Field, childTerms.ToArray()) ?? node.Field; - foreach (var child in node.Children) - await child.AcceptAsync(this, context); - } - - public override Task VisitAsync(TermNode node, IQueryVisitorContext context) - { - // using all fields search - if (String.IsNullOrEmpty(node.Field)) - { - return Task.CompletedTask; - } - - node.Field = GetCustomFieldName(node.Field, [node.Term]); - return Task.CompletedTask; - } - - public override Task VisitAsync(TermRangeNode node, IQueryVisitorContext context) - { - node.Field = GetCustomFieldName(node.Field, [node.Min, node.Max]); - return Task.CompletedTask; - } - - public override Task VisitAsync(ExistsNode node, IQueryVisitorContext context) - { - node.Field = GetCustomFieldName(node.Field, []); - return Task.CompletedTask; - } - - public override Task VisitAsync(MissingNode node, IQueryVisitorContext context) - { - node.Field = GetCustomFieldName(node.Field, []); - return Task.CompletedTask; - } - - private string? GetCustomFieldName(string field, string[] terms) - { - if (String.IsNullOrEmpty(field)) - return null; - - string[] parts = field.Split('.'); - if (parts.Length != 2 || (parts.Length == 2 && parts[1].StartsWith("@"))) - return field; - - if (String.Equals(parts[0], "data", StringComparison.OrdinalIgnoreCase)) - { - string termType; - if (String.Equals(parts[1], Event.KnownDataKeys.SessionEnd, StringComparison.OrdinalIgnoreCase)) - termType = "d"; - else if (String.Equals(parts[1], Event.KnownDataKeys.SessionHasError, StringComparison.OrdinalIgnoreCase)) - termType = "b"; - else - termType = GetTermType(terms); - - field = $"idx.{parts[1].ToLowerInvariant()}-{termType}"; - } - else if (String.Equals(parts[0], "ref", StringComparison.OrdinalIgnoreCase)) - { - field = $"idx.{parts[1].ToLowerInvariant()}-r"; - } - - return field; - } - - private static string GetTermType(string[] terms) - { - string termType = "s"; - - var trimmedTerms = terms.Where(t => t is not null).Distinct().ToList(); - foreach (string term in trimmedTerms) - { - if (term.StartsWith("*")) - continue; - - if (Boolean.TryParse(term, out bool boolResult)) - termType = "b"; - else if (term.IsNumeric()) - termType = "n"; - else if (DateTime.TryParse(term, out var dateResult)) - termType = "d"; - - break; - } - - // Some terms can be a string date range: [now TO now/d+1d} - if (String.Equals(termType, "s") && trimmedTerms.Count > 0 && trimmedTerms.All(t => String.Equals(t, "now", StringComparison.OrdinalIgnoreCase) || t.StartsWith("now/", StringComparison.OrdinalIgnoreCase))) - termType = "d"; - - return termType; - } - - public static Task RunAsync(IQueryNode node, IQueryVisitorContext? context = null) - { - return new EventFieldsQueryVisitor().AcceptAsync(node, context); - } - - public static IQueryNode Run(IQueryNode node, IQueryVisitorContext? context = null) - { - return RunAsync(node, context).GetAwaiter().GetResult(); - } -} diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index 7b9abeb862..e332e6edea 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -24,7 +24,8 @@ public Task InitializeAsync() builder.AddElasticsearch("Elasticsearch", port: 9200) .WithContainerName("Exceptionless-Elasticsearch-Test") - .WithLifetime(ContainerLifetime.Persistent); + .WithLifetime(ContainerLifetime.Persistent) + .WithKibana(b => b.WithLifetime(ContainerLifetime.Persistent).WithContainerName("Exceptionless-Kibana-Test")); _app = builder.Build(); diff --git a/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs index 226264e83b..81c0da4fa3 100644 --- a/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs @@ -11,6 +11,7 @@ using Nest; using Xunit; using Xunit.Abstractions; +using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Tests.Migrations; @@ -39,6 +40,7 @@ protected override void RegisterServices(IServiceCollection services) [Fact] public async Task WillMergeDuplicatedStacks() { + Log.DefaultMinimumLevel = LogLevel.Trace; var originalStack = _stackData.GenerateStack(); originalStack.Id = ObjectId.GenerateNewId().ToString(); originalStack.TotalOccurrences = 100; diff --git a/tests/Exceptionless.Tests/Search/EventIndexTests.cs b/tests/Exceptionless.Tests/Search/EventIndexTests.cs index 5a66fab4d9..3c90e8c196 100644 --- a/tests/Exceptionless.Tests/Search/EventIndexTests.cs +++ b/tests/Exceptionless.Tests/Search/EventIndexTests.cs @@ -1,6 +1,7 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Utility; using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Models; @@ -446,6 +447,6 @@ private async Task> GetByFilterAsync(string filter, var result = await _validator.ValidateQueryAsync(filter); Assert.True(result.IsValid); Log.SetLogLevel(LogLevel.Trace); - return await _repository.FindAsync(q => q.FilterExpression(filter)); + return await _repository.FindAsync(q => q.Organization(SampleDataService.TEST_ORG_ID).FilterExpression(filter)); } } diff --git a/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs b/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs index 88deffe4bd..632415cd64 100644 --- a/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs +++ b/tests/Exceptionless.Tests/Search/PersistentEventQueryValidatorTests.cs @@ -69,8 +69,6 @@ public async Task CanProcessQueryAsync(string query, string expected, bool isVal return; } - // NOTE: we have to do this because we don't have access to the right query parser instance. - result = await EventFieldsQueryVisitor.RunAsync(result, context); Assert.Equal(expected, await GenerateQueryVisitor.RunAsync(result, context)); var info = await _validator.ValidateQueryAsync(result);