EF-149: Support GroupBy aggregation

Translate single-collection GroupBy to MongoDB $group via the driver's
LINQ provider. TranslateGroupBy now returns a GroupByShaperExpression
which TranslateSelect / the projection binder collapse into a scalar or
anonymous projection of the key and aggregates; the captured chain the
driver receives already renders to $group/$project.

Supports scalar/composite/anonymous keys, element and result selectors,
the standard aggregates (Count/LongCount/Sum/Min/Max/Average), and
post-group Where/OrderBy/Select/Distinct.

GroupBy composed with Join/GroupJoin/subqueries, grouping by a
cross-collection navigation, and per-group entity (IGrouping)
materialization remain unsupported and are rejected cleanly. Joining
over a grouped query previously returned silent wrong results; it now
fails translation via the IsGroupByQuery guard.

Flips the NorthwindGroupByQueryMongoTest conformance suite from asserting
translation failure to asserting the generated $group pipeline (green on
EF8/EF9/EF10), and updates the affected SetOperations/KeylessEntities/
AggregateOperators/Miscellaneous overrides. EF-149 failing-spec count
246 -> 115.

Author: damieng

docs/failing-spec-tests.md

@@ -25,7 +25,7 @@ that currently lack a ticket. Counts are sourced from `tests/MongoDB.EntityFrame
 | Ticket | Comment subject | Description | Count |
 | --- | --- | --- | --- |
 | [EF-117](https://jira.mongodb.org/browse/EF-117) | _(no remaining `// Fails:` tags)_ | Cross-collection **Include**/`ThenInclude` is now implemented for the tested shapes. The five tests formerly tagged here were re-investigated: `Outer_identifier_correctly_determined_when_doing_include_on_right_side_of_left_join` (tracking + no-tracking) now **passes**; `Collection_include_over_result_of_single_non_scalar` and `Do_not_erase_projection_mapping_when_adding_single_projection` actually fail on cross-`DbSet` subquery translation (re-tagged **EF-X001**); `Included_one_to_many_query_with_client_eval` fails on driver client-evaluation (re-tagged **EF-X003**); and Include on a keyless entity (incl. multi-level) is a genuine PK-less `$lookup` gap (re-tagged **EF-X019**). EF-117 no longer has any active `// Fails:` tags. (Join/GroupJoin/SelectMany/RightJoin/subquery failures formerly tagged here were re-categorized — see EF-X001/EF-216/EF-220/EF-X016/EF-X017/EF-X018.) | 0 |
-| [EF-149](https://jira.mongodb.org/browse/EF-149) | `GroupBy issue EF-149` | `GroupBy` translation is severely limited; most non-trivial group-by shapes fail to translate. | 246 |
+| [EF-149](https://jira.mongodb.org/browse/EF-149) | `GroupBy issue EF-149` | Single-collection `GroupBy` (scalar/composite/anonymous keys, element/result selectors, `Count`/`LongCount`/`Sum`/`Min`/`Max`/`Average`, and post-group `Where`/`OrderBy`/`Select`/`Distinct`) now translates to `$group`. Remaining gaps: `GroupBy` composed with `Join`/`GroupJoin`/subqueries, grouping by a cross-collection navigation, and per-group entity (`IGrouping`) materialization. | 115 |
 | [EF-153](https://jira.mongodb.org/browse/EF-153) | `TagWith EF-153` | `TagWith(...)` content is silently dropped — does not appear in the emitted MQL. | 9 |
 | [EF-164](https://jira.mongodb.org/browse/EF-164) | `Missing property values issue EF-164` / `Projections issue EF-164` | BSON documents that omit a required scalar (or required navigation) throw on materialization — `Project_root_with_missing_scalars`, `Project_root_entity_with_missing_required_navigation`, etc. | 3 |
 | [EF-202](https://jira.mongodb.org/browse/EF-202) | `Entity equality issue EF-202` | Comparing two entities (`entity1 == entity2` / `Contains(entity)`) is not lowered to a key-equality comparison. | 4 |

src/MongoDB.EntityFrameworkCore/Query/Expressions/MongoQueryExpression.cs

@@ -51,6 +51,12 @@ _projectionMapping[new ProjectionMember()] =
     /// </summary>
     public Expression? CapturedExpression { get; set; }
 
+    /// <summary>
+    /// Set once this query has had a <c>GroupBy</c> applied. Composing a join over a grouped query is not
+    /// yet supported and produces wrong results if attempted, so the join translators reject it.
+    /// </summary>
+    public bool IsGroupByQuery { get; set; }
+
     /// <inheritdoc />
     public override Type Type
         => typeof(object);

src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoProjectionBindingExpressionVisitor.cs

@@ -114,6 +114,11 @@ public override Expression Visit(Expression expression)
             case ConstantExpression:
                 return expression;
 
+            // g.Key over a grouping: bind against the (already root-rebound) key selector so a scalar key
+            // folds to a root member and a composite/anonymous key is walked by VisitNew.
+            case MemberExpression { Expression: GroupByShaperExpression groupByShaper, Member.Name: "Key" }:
+                return Visit(groupByShaper.KeySelector);
+
             case MemberExpression memberExpression:
                 var currentProjectionMember = GetCurrentProjectionMember();
                 _projectionMapping[currentProjectionMember] = memberExpression;
@@ -127,6 +132,17 @@ when IsScalarMethodPropertyAccess(methodCallExpression):
 
                 return new ProjectionBindingExpression(_queryExpression, projMember, expression.Type);
 
+            // An aggregate over a grouping (g.Count(), g.Sum(o => o.X), g.Min/Max/Average/LongCount).
+            // A selector-bearing aggregate is lowered to Sum(Select(grouping, sel)), so the grouping is
+            // reached by unwrapping the Enumerable/Queryable source chain rather than being Arguments[0].
+            // The driver renders the actual accumulator from the captured chain; we only need a scalar
+            // projection binding so the shaper reads the aggregated field from the result document.
+            case MethodCallExpression aggregateCall when IsGroupByAggregate(aggregateCall):
+                var aggMember = GetCurrentProjectionMember();
+                _projectionMapping[aggMember] = aggregateCall;
+
+                return new ProjectionBindingExpression(_queryExpression, aggMember, expression.Type);
+
             default:
                 return base.Visit(expression);
         }
@@ -636,6 +652,50 @@ private void ExitProjectionMember()
     /// being fully visited. This covers <c>EF.Property</c> (for non-navigation properties) and
     /// <c>Mql.Field</c> calls.
     /// </summary>
+    /// <summary>
+    /// Whether a method call is a scalar aggregate terminal (<c>Count</c>/<c>LongCount</c>/<c>Sum</c>/
+    /// <c>Min</c>/<c>Max</c>/<c>Average</c>) whose source chain roots at a grouping. Selector-bearing
+    /// aggregates are lowered to <c>Sum(Select(grouping, sel))</c>, so the grouping is found by unwrapping
+    /// the <see cref="Enumerable"/>/<see cref="Queryable"/> source chain.
+    /// </summary>
+    private static bool IsGroupByAggregate(MethodCallExpression call)
+    {
+        if (call.Arguments.Count == 0)
+        {
+            return false;
+        }
+
+        switch (call.Method.Name)
+        {
+            case nameof(Enumerable.Count):
+            case nameof(Enumerable.LongCount):
+            case nameof(Enumerable.Sum):
+            case nameof(Enumerable.Min):
+            case nameof(Enumerable.Max):
+            case nameof(Enumerable.Average):
+                break;
+            default:
+                return false;
+        }
+
+        var source = call.Arguments[0];
+        while (true)
+        {
+            switch (source)
+            {
+                case GroupByShaperExpression:
+                    return true;
+                case MethodCallExpression { Arguments.Count: > 0 } inner
+                    when inner.Method.DeclaringType == typeof(Enumerable)
+                         || inner.Method.DeclaringType == typeof(Queryable):
+                    source = inner.Arguments[0];
+                    continue;
+                default:
+                    return false;
+            }
+        }
+    }
+
     private static bool IsScalarMethodPropertyAccess(MethodCallExpression methodCallExpression)
     {
         if (methodCallExpression.TryGetEFPropertyArguments(out var source, out var memberName))

src/MongoDB.EntityFrameworkCore/Query/Visitors/MongoQueryableMethodTranslatingExpressionVisitor.cs

@@ -115,21 +115,26 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
                 case nameof(Queryable.Max) when methodDefinition == QueryableMethods.MaxWithoutSelector
                                                 || methodDefinition == QueryableMethods.MaxWithSelector:
 
-                // Join operations - delegate to base class which calls our Translate* overrides
+                // Join / GroupBy operations - delegate to base class which calls our Translate* overrides
                 case nameof(Queryable.Join) when methodDefinition == QueryableMethods.Join:
                 case nameof(Queryable.GroupJoin) when methodDefinition == QueryableMethods.GroupJoin:
 #if !EF8 && !EF9
                 case nameof(Queryable.LeftJoin) when methodDefinition == QueryableMethods.LeftJoin:
 #endif
                 case nameof(Queryable.DefaultIfEmpty) when methodDefinition == QueryableMethods.DefaultIfEmptyWithArgument
                                                            || methodDefinition == QueryableMethods.DefaultIfEmptyWithoutArgument:
+                // All four GroupBy overloads route to TranslateGroupBy; unsupported shapes (element/result
+                // selectors) return null there and fail cleanly, rather than falling through to the no-op
+                // capture path below.
+                case nameof(Queryable.GroupBy) when methodDefinition == QueryableMethods.GroupByWithKeySelector
+                                                    || methodDefinition == QueryableMethods.GroupByWithKeyElementSelector
+                                                    || methodDefinition == QueryableMethods.GroupByWithKeyResultSelector
+                                                    || methodDefinition == QueryableMethods.GroupByWithKeyElementResultSelector:
 
                 // Operations not supported, but we want to bubble through for better error messages
 #if !EF8 && !EF9
                 case nameof(Queryable.RightJoin) when methodDefinition == QueryableMethods.RightJoin:
 #endif
-                case nameof(Queryable.GroupBy) when methodDefinition == QueryableMethods.GroupByWithKeySelector
-                                                    || methodDefinition == QueryableMethods.GroupByWithKeyElementSelector:
                 case nameof(Queryable.Contains) when methodDefinition == QueryableMethods.Contains:
                 case nameof(Queryable.Except) when methodDefinition == QueryableMethods.Except:
                 case nameof(Queryable.Intersect) when methodDefinition == QueryableMethods.Intersect:
@@ -579,7 +584,31 @@ protected override QueryableMethodTranslatingExpressionVisitor CreateSubqueryVis
 
     protected override ShapedQueryExpression? TranslateGroupBy(ShapedQueryExpression source, LambdaExpression keySelector,
         LambdaExpression? elementSelector, LambdaExpression? resultSelector)
-        => null;
+    {
+        // The driver's LINQ provider renders the captured GroupBy(...).Select(...) chain to a $group
+        // stage itself; our job is only the EF-side shaper. We represent the grouping with EF's
+        // GroupByShaperExpression, which TranslateSelect / the projection binder collapse into a scalar
+        // or anonymous projection of the key and aggregates.
+        //
+        // Element and result selectors are not yet supported (later phases of EF-149); returning null
+        // for them produces EF's canonical "could not be translated" message.
+        if (elementSelector != null || resultSelector != null)
+            return null;
+
+        // Grouping over a join/lookup source (GroupBy after Join/GroupJoin, or grouping by a cross-collection
+        // navigation key that nav-expansion turned into a $lookup) is not yet supported. Attempting it either
+        // produces wrong results or throws deep in shaping ("Property 'Key' is not defined for IGrouping<.,
+        // LeftJoinResult<..>>"); reject cleanly so EF reports "could not be translated".
+        var mongoQueryExpression = (MongoQueryExpression)source.QueryExpression;
+        if (mongoQueryExpression.IsJoinQuery || mongoQueryExpression.GetPendingLookups().Count > 0)
+            return null;
+
+        var keyShaper = ReplacingExpressionVisitor.Replace(
+            keySelector.Parameters.Single(), source.ShaperExpression, keySelector.Body);
+
+        mongoQueryExpression.IsGroupByQuery = true;
+        return source.UpdateShaperExpression(new GroupByShaperExpression(keyShaper, source));
+    }
 
     protected override ShapedQueryExpression? TranslateGroupJoin(ShapedQueryExpression outer, ShapedQueryExpression inner,
         LambdaExpression outerKeySelector, LambdaExpression innerKeySelector, LambdaExpression resultSelector)
@@ -609,6 +638,12 @@ protected override QueryableMethodTranslatingExpressionVisitor CreateSubqueryVis
         var outerQueryExpression = (MongoQueryExpression)outer.QueryExpression;
         var innerQueryExpression = (MongoQueryExpression)inner.QueryExpression;
 
+        // Joining over a grouped query (e.g. GroupBy(...).Select(...) on either side of a Join/GroupJoin) is
+        // not yet supported; the driver can't render Join against a $group pipeline and silently returns
+        // wrong results. Reject so EF reports the query as untranslatable.
+        if (outerQueryExpression.IsGroupByQuery || innerQueryExpression.IsGroupByQuery)
+            return null;
+
         outerQueryExpression.AddInnerCollection(innerQueryExpression.CollectionExpression.EntityType);
 
         // Rebind the inner entity's projection to the outer MongoQueryExpression.

tests/MongoDB.EntityFrameworkCore.FunctionalTests/Query/ProjectionTests.cs

@@ -1125,13 +1125,18 @@ public void Select_projection_nested_collection_to_list()
     }
 
     [Fact]
-    public void Select_projection_group_by_not_supported()
+    public void Select_projection_group_by()
     {
-        Assert.ThrowsAny<Exception>(() =>
-            _db.Planets
-                .GroupBy(p => p.hasRings)
-                .Select(g => new { g.Key, Count = g.Count() })
-                .ToList());
+        var results = _db.Planets
+            .GroupBy(p => p.hasRings)
+            .Select(g => new { g.Key, Count = g.Count() })
+            .OrderBy(r => r.Key)
+            .ToList();
+
+        // Planets group into exactly two buckets (with and without rings); every planet is counted once.
+        Assert.Equal(2, results.Count);
+        Assert.Equal(_db.Planets.Count(), results.Sum(r => r.Count));
+        Assert.All(results, r => Assert.True(r.Count > 0));
     }
 
     private class OrderWithDates

tests/MongoDB.EntityFrameworkCore.SpecificationTests/Query/NorthwindAggregateOperatorsQueryMongoTest.cs

@@ -2163,8 +2163,12 @@ public override async Task Not_Any_false(bool async)
 
     public override async Task Contains_inside_aggregate_function_with_GroupBy(bool async)
     {
-        // Fails: GroupBy issue EF-149
-        await AssertTranslationFailed(() => base.Contains_inside_aggregate_function_with_GroupBy(async));
+        await base.Contains_inside_aggregate_function_with_GroupBy(async);
+
+        AssertMql(
+            """
+Customers.{ "$group" : { "_id" : "$Country", "__agg0" : { "$sum" : { "$cond" : { "if" : { "$in" : ["$City", ["London", "Berlin"]] }, "then" : 1, "else" : 0 } } } } }, { "$project" : { "_v" : "$__agg0", "_id" : 0 } }
+""");
     }
 
     public override async Task Contains_inside_Average_without_GroupBy(bool async)