.Net: [MEVD] [SQL Server] Support DateTimeOffset and string arrays (#13269)

Closes #13268

/cc @yorek @uc-msft (this uses JSON and JSON_CONTAINS() to map
`string[]` to the database and perform Contains over them.

Author: roji

dotnet/src/VectorData/SqlServer/SqlServerCommandBuilder.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Linq.Expressions;
 using System.Text;
+using System.Text.Json;
 using Microsoft.Data.SqlClient;
 using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.AI;
@@ -57,7 +58,15 @@ internal static SqlCommand CreateTable(
         {
             if (dataProperty.IsIndexed)
             {
-                sb.AppendFormat("CREATE INDEX ");
+                var sqlType = Map(dataProperty);
+                if (sqlType == "JSON")
+                {
+                    sb.AppendFormat("CREATE JSON INDEX ");
+                }
+                else
+                {
+                    sb.AppendFormat("CREATE INDEX ");
+                }
                 sb.AppendIndexName(tableName, dataProperty.StorageName);
                 sb.AppendFormat(" ON ").AppendTableName(schema, tableName);
                 sb.AppendFormat("([{0}]);", dataProperty.StorageName);
@@ -621,6 +630,13 @@ private static void AddParameter(this SqlCommand command, PropertyModel? propert
                 command.Parameters.AddWithValue(name, new SqlVector<float>(vectorArray));
                 break;
 
+            case string[] strings:
+                command.Parameters.AddWithValue(name, JsonSerializer.Serialize(strings, SqlServerJsonSerializerContext.Default.StringArray));
+                break;
+            case List<string> strings:
+                command.Parameters.AddWithValue(name, JsonSerializer.Serialize(strings, SqlServerJsonSerializerContext.Default.ListString));
+                break;
+
             default:
                 command.Parameters.AddWithValue(name, value);
                 break;
@@ -641,6 +657,7 @@ private static string Map(PropertyModel property)
             Type t when t == typeof(byte[]) => "VARBINARY(MAX)",
             Type t when t == typeof(bool) => "BIT",
             Type t when t == typeof(DateTime) => "DATETIME2",
+            Type t when t == typeof(DateTimeOffset) => "DATETIMEOFFSET",
 #if NET
             Type t when t == typeof(DateOnly) => "DATE",
             Type t when t == typeof(TimeOnly) => "TIME",
@@ -649,6 +666,8 @@ private static string Map(PropertyModel property)
             Type t when t == typeof(double) => "FLOAT",
             Type t when t == typeof(float) => "REAL",
 
+            Type t when t == typeof(string[]) || t == typeof(List<string>) => "JSON",
+
             _ => throw new NotSupportedException($"Type {property.Type} is not supported.")
         };
 

dotnet/src/VectorData/SqlServer/SqlServerFilterTranslator.cs

@@ -52,6 +52,7 @@ protected override void TranslateConstant(object? value)
                     : string.Format(CultureInfo.InvariantCulture, @"'{0:HH\:mm\:ss\.FFFFFFF}'", value));
                 return;
 #endif
+
             default:
                 base.TranslateConstant(value);
                 break;
@@ -72,7 +73,18 @@ protected override void GenerateColumn(PropertyModel property, bool isSearchCond
     }
 
     protected override void TranslateContainsOverArrayColumn(Expression source, Expression item)
-        => throw new NotSupportedException("Unsupported Contains expression");
+    {
+        if (item.Type != typeof(string))
+        {
+            throw new NotSupportedException("Unsupported Contains expression");
+        }
+
+        this._sql.Append("JSON_CONTAINS(");
+        this.Translate(source);
+        this._sql.Append(", ");
+        this.Translate(item);
+        this._sql.Append(") = 1");
+    }
 
     protected override void TranslateContainsOverParameterizedArray(Expression source, Expression item, object? value)
     {

dotnet/src/VectorData/SqlServer/SqlServerJsonSerializerContext.cs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Connectors.SqlServer;
+
+// For mapping string[] properties to SQL Server JSON columns
+[JsonSerializable(typeof(string[]))]
+[JsonSerializable(typeof(List<string>))]
+internal partial class SqlServerJsonSerializerContext : JsonSerializerContext;

dotnet/src/VectorData/SqlServer/SqlServerMapper.cs

@@ -1,8 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Runtime.InteropServices;
+using System.Text.Json;
 using Microsoft.Data.SqlClient;
 using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.AI;
@@ -112,7 +114,9 @@ static void PopulateValue(SqlDataReader reader, PropertyModel property, object r
                     case var t when t == typeof(DateTime):
                         property.SetValue(record, reader.GetDateTime(ordinal)); // DATETIME2
                         break;
-
+                    case var t when t == typeof(DateTimeOffset):
+                        property.SetValue(record, reader.GetDateTimeOffset(ordinal)); // DATETIMEOFFSET
+                        break;
 #if NET
                     case var t when t == typeof(DateOnly):
                         property.SetValue(record, reader.GetFieldValue<DateOnly>(ordinal)); // DATE
@@ -122,6 +126,18 @@ static void PopulateValue(SqlDataReader reader, PropertyModel property, object r
                         break;
 #endif
 
+                    // We map string[] and List<string> properties to SQL Server JSON columns, so deserialize from JSON here.
+                    case var t when t == typeof(string[]):
+                        property.SetValue(record, JsonSerializer.Deserialize<string[]>(
+                            reader.GetString(ordinal),
+                            SqlServerJsonSerializerContext.Default.StringArray));
+                        break;
+                    case var t when t == typeof(List<string>):
+                        property.SetValue(record, JsonSerializer.Deserialize<List<string>>(
+                            reader.GetString(ordinal),
+                            SqlServerJsonSerializerContext.Default.ListString));
+                        break;
+
                     default:
                         throw new NotSupportedException($"Unsupported type '{property.Type.Name}' for property '{property.ModelName}'.");
                 }

dotnet/src/VectorData/SqlServer/SqlServerModelBuilder.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.AI;
@@ -33,7 +34,7 @@ protected override bool IsKeyPropertyTypeValid(Type type, [NotNullWhen(false)] o
 
     protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
     {
-        supportedTypes = "string, short, int, long, double, float, decimal, bool, DateTime, DateTimeOffset, DateOnly, TimeOnly, Guid, byte[]";
+        supportedTypes = "string, short, int, long, double, float, decimal, bool, DateTime, DateTimeOffset, DateOnly, TimeOnly, Guid, byte[], string[], List<string>";
 
         if (Nullable.GetUnderlyingType(type) is Type underlyingType)
         {
@@ -49,6 +50,7 @@ protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)]
             || type == typeof(byte[]) // VARBINARY
             || type == typeof(bool) // BIT
             || type == typeof(DateTime) // DATETIME2
+            || type == typeof(DateTimeOffset) // DATETIMEOFFSET
 #if NET
             || type == typeof(DateOnly) // DATE
                                         // We don't support mapping TimeSpan to TIME on purpose
@@ -57,7 +59,11 @@ protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)]
 #endif
             || type == typeof(decimal) // DECIMAL
             || type == typeof(double) // FLOAT
-            || type == typeof(float); // REAL
+            || type == typeof(float) // REAL
+
+            // We map string[] to the SQL Server 2025 JSON data type (anyone using vector search is already using 2025)
+            || type == typeof(string[]) // JSON
+            || type == typeof(List<string>); // JSON
     }
 
     protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)

dotnet/test/VectorData/SqlServer.ConformanceTests/Filter/SqlServerBasicFilterTests.cs

@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using Microsoft.Extensions.VectorData;
 using SqlServer.ConformanceTests.Support;
 using VectorData.ConformanceTests.Filter;
 using VectorData.ConformanceTests.Support;
@@ -36,25 +35,6 @@ await this.TestFilterAsync(
             r => r["String"] != null && r["String"] != "foo");
     }
 
-    public override Task Contains_over_field_string_array()
-        => Assert.ThrowsAsync<InvalidOperationException>(() => base.Contains_over_field_string_array());
-
-    public override Task Contains_over_field_string_List()
-        => Assert.ThrowsAsync<InvalidOperationException>(() => base.Contains_over_field_string_List());
-
-    public override Task Contains_with_Enumerable_Contains()
-        => Assert.ThrowsAsync<InvalidOperationException>(() => base.Contains_with_Enumerable_Contains());
-
-#if !NETFRAMEWORK
-    public override Task Contains_with_MemoryExtensions_Contains()
-        => Assert.ThrowsAsync<InvalidOperationException>(() => base.Contains_with_MemoryExtensions_Contains());
-#endif
-
-#if NET10_0_OR_GREATER
-    public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer()
-        => Assert.ThrowsAsync<InvalidOperationException>(() => base.Contains_with_MemoryExtensions_Contains_with_null_comparer());
-#endif
-
     [Fact(Skip = "Not supported")]
     [Obsolete("Legacy filters are not supported")]
     public override Task Legacy_And() => throw new NotSupportedException();
@@ -78,12 +58,5 @@ public override Task Contains_with_MemoryExtensions_Contains_with_null_comparer(
         public override TestStore TestStore => SqlServerTestStore.Instance;
 
         public override string CollectionName => s_uniqueName;
-
-        // Override to remove the string collection properties, which aren't (currently) supported on SqlServer
-        public override VectorStoreCollectionDefinition CreateRecordDefinition()
-            => new()
-            {
-                Properties = base.CreateRecordDefinition().Properties.Where(p => p.Type != typeof(string[]) && p.Type != typeof(List<string>)).ToList()
-            };
     }
 }

dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerDataTypeTests.cs

@@ -10,14 +10,16 @@ namespace SqlServer.ConformanceTests;
 public class SqlServerDataTypeTests(SqlServerDataTypeTests.Fixture fixture)
     : DataTypeTests<Guid, DataTypeTests<Guid>.DefaultRecord>(fixture), IClassFixture<SqlServerDataTypeTests.Fixture>
 {
+    public override Task String_array()
+        => this.Test<string[]>(
+            "StringArray",
+            ["foo", "bar"],
+            ["foo", "baz"],
+            // SQL Server doesn't support comparing JSON
+            isFilterable: false);
+
     public new class Fixture : DataTypeTests<Guid, DataTypeTests<Guid>.DefaultRecord>.Fixture
     {
         public override TestStore TestStore => SqlServerTestStore.Instance;
-
-        public override Type[] UnsupportedDefaultTypes { get; } =
-        [
-            typeof(DateTimeOffset),
-            typeof(string[])
-        ];
     }
 }