.Net: [MEVD] Go over all key types, add support for Guid (#13315)

* Added new KeyTypeTests to cover provider key support, alongside the
existing DataTypeTests and EmbeddingTypeTests; moved all three into
TypeTests folder.
* As discussed offline, this adds Guid-as-string key support for all
providers. From research this seems to be very standard practice across
a wide range of databases: JSON-based databases generally encode various
types (e.g. DateTime) as string, and Guids are standard there. Sqlite is
similar, with Microsoft.Data.Sqlite supporting it natively. This notably
allows upper layers to assume that they can always use Guid, which helps
MEDI ([see this
issue](dotnet/extensions#6973)).
* Per-provider notes
* Azure AI Search: string only supported, added Guid-as-string as an
option
([docs](https://learn.microsoft.com/en-us/rest/api/searchservice/data-type-map-for-indexers-in-azure-search#bkmk_sql_search)
encouraging mapping Guid to strings)
* MongoDB/CosmosMongoDB: Added int and long support (Guid was already
supported).
    * CosmosNoSql: Added Guid-as-string (standard practice)
    * Pinecone: Added Guid-as-string
    * Redis: Added Guid-as-string
* SQL Server: removed DateTime and `byte[]` as key types (as above) and
added Guid.
* SQLite: The Microsoft.Data.Sqlite (the low-level ADO.NET provider
we're built on top of) supports GUIDs as TEXT; so it's really not us
deciding to support - it's the lower-level driver. In addition, we
support mapping boolean to INTEGER (1/0), which seems similar. Added
support as data property as well, not just key.

Closes #11784
Closes #12182

Author: roji

dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs

@@ -44,9 +44,9 @@ protected override void ProcessTypeProperties(Type type, VectorStoreCollectionDe
 
     protected override bool IsKeyPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
     {
-        supportedTypes = "string, Guid, ObjectId";
+        supportedTypes = "string, int, long, Guid, ObjectId";
 
-        return type == typeof(string) || type == typeof(Guid) || type == typeof(ObjectId);
+        return type == typeof(string) || type == typeof(int) || type == typeof(long) || type == typeof(Guid) || type == typeof(ObjectId);
     }
 
     protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)

dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs

@@ -83,9 +83,9 @@ internal AzureAISearchCollection(SearchIndexClient searchIndexClient, string nam
         Verify.NotNull(searchIndexClient);
         Verify.NotNullOrWhiteSpace(name);
 
-        if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(object))
+        if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object))
         {
-            throw new NotSupportedException("Only string keys are supported.");
+            throw new NotSupportedException("Only string and Guid keys are supported.");
         }
 
         options ??= AzureAISearchCollectionOptions.Default;
@@ -791,7 +791,13 @@ private string GetStringKey(TKey key)
     {
         Verify.NotNull(key);
 
-        var stringKey = key as string ?? throw new UnreachableException("string key should have been validated during model building");
+        var stringKey = key switch
+        {
+            string s => s,
+            Guid g => g.ToString(),
+
+            _ => throw new UnreachableException("string key should have been validated during model building")
+        };
 
         Verify.NotNullOrWhiteSpace(stringKey, nameof(key));
 

dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs

@@ -31,9 +31,9 @@ protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)
 
     internal static bool IsKeyPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)
     {
-        supportedTypes = "string";
+        supportedTypes = "string, Guid";
 
-        return type == typeof(string);
+        return type == typeof(string) || type == typeof(Guid);
     }
 
     internal static bool IsDataPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)

dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs

@@ -67,6 +67,8 @@ public class CosmosMongoCollection<TKey, TRecord> : VectorStoreCollection<TKey,
     /// <summary>The size of the dynamic candidate list for search.</summary>
     private readonly int _efSearch;
 
+    private static readonly Type[] s_validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId), typeof(int), typeof(long)];
+
     /// <summary>
     /// Initializes a new instance of the <see cref="CosmosMongoCollection{TKey, TRecord}"/> class.
     /// </summary>
@@ -95,9 +97,9 @@ internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func<C
         Verify.NotNull(mongoDatabase);
         Verify.NotNullOrWhiteSpace(name);
 
-        if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(object))
+        if (!s_validKeyTypes.Contains(typeof(TKey)) && typeof(TKey) != typeof(object))
         {
-            throw new NotSupportedException("Only string keys are supported.");
+            throw new NotSupportedException("Only ObjectID, string, Guid, int and long keys are supported.");
         }
 
         options ??= CosmosMongoCollectionOptions.Default;

dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs

@@ -134,9 +134,12 @@ internal CosmosNoSqlCollection(
     {
         try
         {
-            if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(CosmosNoSqlCompositeKey) && typeof(TKey) != typeof(object))
+            if (typeof(TKey) != typeof(string)
+                && typeof(TKey) != typeof(Guid)
+                && typeof(TKey) != typeof(CosmosNoSqlCompositeKey)
+                && typeof(TKey) != typeof(object))
             {
-                throw new NotSupportedException($"Only {nameof(String)} and {nameof(CosmosNoSqlCompositeKey)} keys are supported.");
+                throw new NotSupportedException($"Only string, Guid and {nameof(CosmosNoSqlCompositeKey)} keys are supported.");
             }
 
             this._database = databaseProvider(clientWrapper.Client);
@@ -852,13 +855,22 @@ private static IEnumerable<CosmosNoSqlCompositeKey> GetCompositeKeys(IEnumerable
         => keys switch
         {
             IEnumerable<CosmosNoSqlCompositeKey> k => k,
+
             IEnumerable<string> k => k.Select(key => new CosmosNoSqlCompositeKey(recordKey: key, partitionKey: key)),
+
+            IEnumerable<Guid> k => k.Select(key =>
+            {
+                var guidString = key.ToString();
+                return new CosmosNoSqlCompositeKey(recordKey: guidString, partitionKey: guidString);
+            }),
+
             IEnumerable<object> k => k.Select(key => key switch
             {
                 string s => new CosmosNoSqlCompositeKey(recordKey: s, partitionKey: s),
                 CosmosNoSqlCompositeKey ck => ck,
                 _ => throw new ArgumentException($"Invalid key type '{key.GetType().Name}'.")
             }),
+
             _ => throw new UnreachableException()
         };
 

dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs

@@ -23,10 +23,9 @@ internal class CosmosNoSqlModelBuilder() : CollectionJsonModelBuilder(s_modelBui
 
     protected override bool IsKeyPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
     {
-        // TODO: Cosmos supports other key types (int, Guid...)
-        supportedTypes = "string";
+        supportedTypes = "string, Guid";
 
-        return type == typeof(string);
+        return type == typeof(string) || type == typeof(Guid);
     }
 
     protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)

dotnet/src/VectorData/MongoDB/MongoCollection.cs

@@ -76,7 +76,7 @@ public class MongoCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRecor
     private readonly int? _numCandidates;
 
     /// <summary>Types of keys permitted.</summary>
-    private readonly Type[] _validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId)];
+    private static readonly Type[] s_validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId), typeof(int), typeof(long)];
 
     /// <summary>
     /// Initializes a new instance of the <see cref="MongoCollection{TKey, TRecord}"/> class.
@@ -106,9 +106,9 @@ internal MongoCollection(IMongoDatabase mongoDatabase, string name, Func<MongoCo
         Verify.NotNull(mongoDatabase);
         Verify.NotNullOrWhiteSpace(name);
 
-        if (!this._validKeyTypes.Contains(typeof(TKey)) && typeof(TKey) != typeof(object))
+        if (!s_validKeyTypes.Contains(typeof(TKey)) && typeof(TKey) != typeof(object))
         {
-            throw new NotSupportedException("Only string, Guid and ObjectID keys are supported.");
+            throw new NotSupportedException("Only ObjectID, string, Guid, int and long keys are supported.");
         }
 
         options ??= MongoCollectionOptions.Default;

dotnet/src/VectorData/Pinecone/PineconeCollection.cs

@@ -76,9 +76,9 @@ internal PineconeCollection(PineconeClient pineconeClient, string name, Func<Pin
         Verify.NotNull(pineconeClient);
         VerifyCollectionName(name);
 
-        if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(object))
+        if (typeof(TKey) != typeof(string) && typeof(TKey) != typeof(Guid) && typeof(TKey) != typeof(object))
         {
-            throw new NotSupportedException("Only string keys are supported.");
+            throw new NotSupportedException("Only string and Guid keys are supported.");
         }
 
         options ??= PineconeCollectionOptions.Default;
@@ -617,7 +617,13 @@ private string GetStringKey(TKey key)
     {
         Verify.NotNull(key);
 
-        var stringKey = key as string ?? throw new UnreachableException("string key should have been validated during model building");
+        var stringKey = key switch
+        {
+            string s => s,
+            Guid g => g.ToString(),
+
+            _ => throw new UnreachableException()
+        };
 
         Verify.NotNullOrWhiteSpace(stringKey, nameof(key));
 

dotnet/src/VectorData/Pinecone/PineconeFilterTranslator.cs

@@ -23,13 +23,21 @@ internal class PineconeFilterTranslator
     private Extensions.VectorData.ProviderServices.CollectionModel _model = null!;
     private ParameterExpression _recordParameter = null!;
 
-    internal Metadata Translate(LambdaExpression lambdaExpression, Extensions.VectorData.ProviderServices.CollectionModel model)
+    internal Metadata? Translate(LambdaExpression lambdaExpression, Extensions.VectorData.ProviderServices.CollectionModel model)
     {
         this._model = model;
 
         Debug.Assert(lambdaExpression.Parameters.Count == 1);
         this._recordParameter = lambdaExpression.Parameters[0];
 
+        // Pinecone doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching
+        // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless),
+        // nor is 'x && true'.
+        if (lambdaExpression.Body is ConstantExpression { Value: true })
+        {
+            return null;
+        }
+
         var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
         var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
 

dotnet/src/VectorData/Pinecone/PineconeMapper.cs

@@ -44,7 +44,12 @@ public Vector MapFromDataToStorageModel(TRecord dataModel, Embedding<float>? gen
         // TODO: what about sparse values?
         var result = new Vector
         {
-            Id = (string)keyObject,
+            Id = keyObject switch
+            {
+                string s => s,
+                Guid g => g.ToString(),
+                _ => throw new UnreachableException()
+            },
             Values = values,
             Metadata = metadata,
             SparseValues = null
@@ -58,7 +63,13 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, bool includeVector
     {
         var outputRecord = model.CreateRecord<TRecord>()!;
 
-        model.KeyProperty.SetValueAsObject(outputRecord, storageModel.Id);
+        model.KeyProperty.SetValueAsObject(outputRecord, model.KeyProperty.Type switch
+        {
+            var t when t == typeof(string) => storageModel.Id,
+            var t when t == typeof(Guid) => Guid.Parse(storageModel.Id),
+
+            _ => throw new UnreachableException()
+        });
 
         if (includeVectors is true)
         {