Skip to content

Commit

Permalink
Adding support for Elasticsearch runtime fields (#68)
Browse files Browse the repository at this point in the history 
* Adding new QueryValidator class. Adding features to ValidationVisitor to allow detecting unresolved fields and other features. Changing time zone to be lazy so that apps don't take a hit getting time zone if the query doesn't need it.

* Progress on runtime-fields. Adding new ElasticFieldResolverVisitor that runs by default and will correct field name casing as well as resolve dynamic runtime fields.

* Progress on runtime field support

* Cleanup code linting. Make timezone an async func that can be lazily loaded.

* Fix async return value issues

* Update ES

* Add UseOptInRuntimeFieldResolver config option

* Update deps

* Update Foundatio

* Fix parsing empty quoted strings Fixes #69
  • Loading branch information
@ejsmith
ejsmith authored Oct 11, 2021
1 parent e7c1490 commit dd34a38
Show file tree
Hide file tree
Showing 47 changed files with 844 additions and 518 deletions.
4 changes: 2 additions & 2 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: '3.5'

services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
environment:
discovery.type: single-node
xpack.security.enabled: 'false'
Expand All @@ -16,7 +16,7 @@ services:
kibana:
depends_on:
- elasticsearch
image: docker.elastic.co/kibana/kibana:7.12.0
image: docker.elastic.co/kibana/kibana:7.14.0
ports:
- 5601:5601
networks:
Expand Down
108 changes: 39 additions & 69 deletions src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ public class ElasticMappingResolver {
private ITypeMapping _serverMapping;
private readonly ITypeMapping _codeMapping;
private readonly Inferrer _inferrer;
private readonly ConcurrentDictionary<string, FieldMapping> _mappingCache = new ConcurrentDictionary<string, FieldMapping>();
private readonly ConcurrentDictionary<string, FieldMapping> _mappingCache = new();
private readonly ILogger _logger;

public static ElasticMappingResolver NullInstance = new ElasticMappingResolver(() => null);
public static ElasticMappingResolver NullInstance = new(() => null);

public ElasticMappingResolver(Func<ITypeMapping> getMapping, Inferrer inferrer = null, ILogger logger = null) {
GetServerMappingFunc = getMapping;
Expand Down Expand Up @@ -77,7 +77,7 @@ public FieldMapping GetMapping(string field, bool followAlias = false) {
if (currentProperties != null)
fieldMapping = currentProperties.Values.FirstOrDefault(m => {
var propertyName = _inferrer.PropertyName(m?.Name);
return propertyName == null ? false : propertyName.Equals(fieldPart, StringComparison.OrdinalIgnoreCase);
return propertyName != null && propertyName.Equals(fieldPart, StringComparison.OrdinalIgnoreCase);
});

// no mapping found, call GetServerMapping again in case it hasn't been called recently and there are possibly new mappings
Expand Down Expand Up @@ -273,67 +273,37 @@ public FieldType GetFieldType(string field) {
if (property?.Type == null)
return FieldType.None;

switch (property.Type) {
case "geo_point":
return FieldType.GeoPoint;
case "geo_shape":
return FieldType.GeoShape;
case "ip":
return FieldType.Ip;
case "binary":
return FieldType.Binary;
case "keyword":
return FieldType.Keyword;
case "string":
case "text":
return FieldType.Text;
case "date":
return FieldType.Date;
case "boolean":
return FieldType.Boolean;
case "completion":
return FieldType.Completion;
case "nested":
return FieldType.Nested;
case "object":
return FieldType.Object;
case "murmur3":
return FieldType.Murmur3Hash;
case "token_count":
return FieldType.TokenCount;
case "percolator":
return FieldType.Percolator;
case "integer":
return FieldType.Integer;
case "long":
return FieldType.Long;
case "short":
return FieldType.Short;
case "byte":
return FieldType.Byte;
case "float":
return FieldType.Float;
case "half_float":
return FieldType.HalfFloat;
case "scaled_float":
return FieldType.ScaledFloat;
case "double":
return FieldType.Double;
case "integer_range":
return FieldType.IntegerRange;
case "float_range":
return FieldType.FloatRange;
case "long_range":
return FieldType.LongRange;
case "double_range":
return FieldType.DoubleRange;
case "date_range":
return FieldType.DateRange;
case "ip_range":
return FieldType.IpRange;
default:
return FieldType.None;
}
return property.Type switch {
"geo_point" => FieldType.GeoPoint,
"geo_shape" => FieldType.GeoShape,
"ip" => FieldType.Ip,
"binary" => FieldType.Binary,
"keyword" => FieldType.Keyword,
"string" or "text" => FieldType.Text,
"date" => FieldType.Date,
"boolean" => FieldType.Boolean,
"completion" => FieldType.Completion,
"nested" => FieldType.Nested,
"object" => FieldType.Object,
"murmur3" => FieldType.Murmur3Hash,
"token_count" => FieldType.TokenCount,
"percolator" => FieldType.Percolator,
"integer" => FieldType.Integer,
"long" => FieldType.Long,
"short" => FieldType.Short,
"byte" => FieldType.Byte,
"float" => FieldType.Float,
"half_float" => FieldType.HalfFloat,
"scaled_float" => FieldType.ScaledFloat,
"double" => FieldType.Double,
"integer_range" => FieldType.IntegerRange,
"float_range" => FieldType.FloatRange,
"long_range" => FieldType.LongRange,
"double_range" => FieldType.DoubleRange,
"date_range" => FieldType.DateRange,
"ip_range" => FieldType.IpRange,
_ => FieldType.None,
};
}

private IProperties MergeProperties(IProperties codeProperties, IProperties serverProperties) {
Expand All @@ -356,7 +326,7 @@ private IProperties MergeProperties(IProperties codeProperties, IProperties serv
if (_inferrer != null) {
// resolve field alias
foreach (var kvp in codeProperties) {
if (!(kvp.Value is IFieldAliasProperty aliasProperty))
if (kvp.Value is not IFieldAliasProperty aliasProperty)
continue;

mergedCodeProperties[kvp.Key] = new FieldAliasProperty {
Expand Down Expand Up @@ -424,7 +394,7 @@ private bool GetServerMapping() {
}

public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, ITypeMapping> mappingBuilder, IElasticClient client, ILogger logger = null) where T : class {
logger = logger ?? NullLogger.Instance;
logger ??= NullLogger.Instance;

return Create(mappingBuilder, client.Infer, () => {
client.Indices.Refresh(Indices.Index<T>());
Expand All @@ -438,7 +408,7 @@ public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, IT
}

public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, ITypeMapping> mappingBuilder, IElasticClient client, string index, ILogger logger = null) where T : class {
logger = logger ?? NullLogger.Instance;
logger ??= NullLogger.Instance;

return Create(mappingBuilder, client.Infer, () => {
client.Indices.Refresh(index);
Expand All @@ -458,7 +428,7 @@ public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, IT
}

public static ElasticMappingResolver Create<T>(IElasticClient client, ILogger logger = null) {
logger = logger ?? NullLogger.Instance;
logger ??= NullLogger.Instance;

return Create(() => {
client.Indices.Refresh(Indices.Index<T>());
Expand All @@ -472,7 +442,7 @@ public static ElasticMappingResolver Create<T>(IElasticClient client, ILogger lo
}

public static ElasticMappingResolver Create(IElasticClient client, string index, ILogger logger = null) {
logger = logger ?? NullLogger.Instance;
logger ??= NullLogger.Instance;

return Create(() => {
client.Indices.Refresh(index);
Expand Down
55 changes: 32 additions & 23 deletions src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,13 @@

namespace Foundatio.Parsers.ElasticQueries {
public class ElasticQueryParser : LuceneQueryParser {
private readonly ElasticQueryParserConfiguration _config;

public ElasticQueryParser(Action<ElasticQueryParserConfiguration> configure = null) {
var config = new ElasticQueryParserConfiguration();
configure?.Invoke(config);
_config = config;
Configuration = config;
}

public ElasticQueryParserConfiguration Configuration => _config;
public ElasticQueryParserConfiguration Configuration { get; }

public override async Task<IQueryNode> ParseAsync(string query, IQueryVisitorContext context = null) {
if (String.IsNullOrEmpty(query))
Expand All @@ -32,34 +30,40 @@ public override async Task<IQueryNode> ParseAsync(string query, IQueryVisitorCon
SetupQueryVisitorContextDefaults(context);
switch (context.QueryType) {
case QueryType.Aggregation:
context.SetMappingResolver(_config.MappingResolver);
result = await _config.AggregationVisitor.AcceptAsync(result, context).ConfigureAwait(false);
context.SetMappingResolver(Configuration.MappingResolver);
result = await Configuration.AggregationVisitor.AcceptAsync(result, context).ConfigureAwait(false);
break;
case QueryType.Query:
context.SetMappingResolver(_config.MappingResolver).SetDefaultFields(_config.DefaultFields);
if (_config.IncludeResolver != null && context.GetIncludeResolver() == null)
context.SetIncludeResolver(_config.IncludeResolver);
context.SetMappingResolver(Configuration.MappingResolver).SetDefaultFields(Configuration.DefaultFields);
if (Configuration.IncludeResolver != null && context.GetIncludeResolver() == null)
context.SetIncludeResolver(Configuration.IncludeResolver);

result = await _config.QueryVisitor.AcceptAsync(result, context).ConfigureAwait(false);
result = await Configuration.QueryVisitor.AcceptAsync(result, context).ConfigureAwait(false);
break;
case QueryType.Sort:
context.SetMappingResolver(_config.MappingResolver);
result = await _config.SortVisitor.AcceptAsync(result, context).ConfigureAwait(false);
context.SetMappingResolver(Configuration.MappingResolver);
result = await Configuration.SortVisitor.AcceptAsync(result, context).ConfigureAwait(false);
break;
}

return result;
}

private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context) {
if (_config.FieldResolver != null && context.GetFieldResolver() == null)
context.SetFieldResolver(_config.FieldResolver);
if (Configuration.EnableRuntimeFieldResolver.HasValue && !context.IsRuntimeFieldResolverEnabled().HasValue)
context.EnableRuntimeFieldResolver(Configuration.EnableRuntimeFieldResolver.Value);

if (Configuration.RuntimeFieldResolver != null && context.GetRuntimeFieldResolver() == null)
context.SetRuntimeFieldResolver(Configuration.RuntimeFieldResolver);

if (_config.MappingResolver != null && context.GetMappingResolver() == null)
context.SetMappingResolver(_config.MappingResolver);
if (Configuration.FieldResolver != null && context.GetFieldResolver() == null)
context.SetFieldResolver(Configuration.FieldResolver);

if (_config.Validator != null && context.GetValidator() == null)
context.SetValidator(_config.Validator);
if (Configuration.MappingResolver != null && context.GetMappingResolver() == null)
context.SetMappingResolver(Configuration.MappingResolver);

if (Configuration.ValidationOptions != null && context.GetValidationOptions() == null)
context.SetValidationOptions(Configuration.ValidationOptions);
}

public async Task<QueryContainer> BuildQueryAsync(string query, IElasticQueryVisitorContext context = null) {
Expand All @@ -71,18 +75,18 @@ public async Task<QueryContainer> BuildQueryAsync(string query, IElasticQueryVis
return await BuildQueryAsync(result, context).ConfigureAwait(false);
}

public Task<QueryContainer> BuildQueryAsync(IQueryNode query, IElasticQueryVisitorContext context = null) {
public async Task<QueryContainer> BuildQueryAsync(IQueryNode query, IElasticQueryVisitorContext context = null) {
if (context == null)
context = new ElasticQueryVisitorContext();

var q = query.GetQuery() ?? new MatchAllQuery();
var q = await query.GetQueryAsync() ?? new MatchAllQuery();
if (context?.UseScoring == false) {
q = new BoolQuery {
Filter = new QueryContainer[] { q }
};
}

return Task.FromResult<QueryContainer>(q);
return q;
}

public async Task<AggregationContainer> BuildAggregationsAsync(string aggregations, IElasticQueryVisitorContext context = null) {
Expand All @@ -94,8 +98,13 @@ public async Task<AggregationContainer> BuildAggregationsAsync(string aggregatio
return await BuildAggregationsAsync(result, context).ConfigureAwait(false);
}

public Task<AggregationContainer> BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null) {
return Task.FromResult<AggregationContainer>(aggregations?.GetAggregation());
#pragma warning disable IDE0060 // Remove unused parameter
public async Task<AggregationContainer> BuildAggregationsAsync(IQueryNode aggregations, IElasticQueryVisitorContext context = null) {
if (aggregations == null)
return null;

#pragma warning restore IDE0060 // Remove unused parameter
return await aggregations?.GetAggregationAsync();
}

public async Task<IEnumerable<IFieldSort>> BuildSortAsync(string sort, IElasticQueryVisitorContext context = null) {
Expand Down
24 changes: 20 additions & 4 deletions src/Foundatio.Parsers.ElasticQueries/ElasticQueryParserConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,17 @@ public ElasticQueryParserConfiguration() {
AddAggregationVisitor(new AssignOperationTypeVisitor(), 0);
AddAggregationVisitor(new CombineAggregationsVisitor(), 10000);
AddVisitor(new FieldResolverQueryVisitor(), 10);
AddVisitor(new ElasticFieldResolverVisitor(), 20);
}

public ILoggerFactory LoggerFactory { get; private set; } = NullLoggerFactory.Instance;
public string[] DefaultFields { get; private set; }
public QueryFieldResolver FieldResolver { get; private set; }
public IncludeResolver IncludeResolver { get; private set; }
public RuntimeFieldResolver RuntimeFieldResolver { get; private set; }
public bool? EnableRuntimeFieldResolver { get; private set; }
public ElasticMappingResolver MappingResolver { get; private set; }
public Func<QueryValidationInfo, Task<bool>> Validator { get; private set; }
public QueryValidationOptions ValidationOptions { get; private set; }
public ChainedQueryVisitor SortVisitor { get; } = new ChainedQueryVisitor();
public ChainedQueryVisitor QueryVisitor { get; } = new ChainedQueryVisitor();
public ChainedQueryVisitor AggregationVisitor { get; } = new ChainedQueryVisitor();
Expand Down Expand Up @@ -69,6 +72,19 @@ public ElasticQueryParserConfiguration UseIncludes(IncludeResolver includeResolv
return AddVisitor(new IncludeVisitor(shouldSkipInclude), priority);
}

public ElasticQueryParserConfiguration UseOptInRuntimeFieldResolver(RuntimeFieldResolver fieldResolver) {
EnableRuntimeFieldResolver = false;
RuntimeFieldResolver = fieldResolver;

return this;
}

public ElasticQueryParserConfiguration UseRuntimeFieldResolver(RuntimeFieldResolver fieldResolver) {
RuntimeFieldResolver = fieldResolver;

return this;
}

public ElasticQueryParserConfiguration UseIncludes(Func<string, string> resolveInclude, ShouldSkipIncludeFunc shouldSkipInclude = null, int priority = 0) {
return UseIncludes(name => Task.FromResult(resolveInclude(name)), shouldSkipInclude, priority);
}
Expand All @@ -77,10 +93,10 @@ public ElasticQueryParserConfiguration UseIncludes(IDictionary<string, string> i
return UseIncludes(name => includes.ContainsKey(name) ? includes[name] : null, shouldSkipInclude, priority);
}

public ElasticQueryParserConfiguration UseValidation(Func<QueryValidationInfo, Task<bool>> validator, int priority = 0) {
Validator = validator;
public ElasticQueryParserConfiguration UseValidation(QueryValidationOptions options, int priority = 30) {
ValidationOptions = options;

return AddVisitor(new ValidationVisitor { ShouldThrow = true }, priority);
return AddVisitor(new ElasticValidationVisitor(), priority);
}

public ElasticQueryParserConfiguration UseNested(int priority = 300) {
Expand Down
Loading

0 comments on commit dd34a38

Please sign in to comment.