Enhance AzureAISearch to support indexes independent of content items (…

…#17556)

src/OrchardCore.Modules/OrchardCore.Search.AzureAI/Drivers/AzureAISearchIndexSettingsDisplayDriver.cs

@@ -0,0 +1,89 @@
+using Microsoft.AspNetCore.Mvc.Rendering;
+using Microsoft.Extensions.Localization;
+using Microsoft.Extensions.Options;
+using OrchardCore.DisplayManagement.Handlers;
+using OrchardCore.DisplayManagement.Views;
+using OrchardCore.Mvc.ModelBinding;
+using OrchardCore.Search.AzureAI.Models;
+using OrchardCore.Search.AzureAI.Services;
+using OrchardCore.Search.AzureAI.ViewModels;
+
+namespace OrchardCore.Search.AzureAI.Drivers;
+
+internal sealed class AzureAISearchIndexSettingsDisplayDriver : DisplayDriver<AzureAISearchIndexSettings>
+{
+    private readonly AzureAISearchIndexManager _indexManager;
+    private readonly AzureAISearchDefaultOptions _azureAIOptions;
+    private readonly IStringLocalizer S;
+
+    public AzureAISearchIndexSettingsDisplayDriver(
+        AzureAISearchIndexManager indexManager,
+        IOptions<AzureAISearchDefaultOptions> azureAIOptions,
+        IStringLocalizer<AzureAISearchIndexSettingsDisplayDriver> stringLocalizer)
+    {
+        _indexManager = indexManager;
+        _azureAIOptions = azureAIOptions.Value;
+        S = stringLocalizer;
+    }
+
+    public override Task<IDisplayResult> DisplayAsync(AzureAISearchIndexSettings settings, BuildDisplayContext context)
+    {
+        return CombineAsync(
+            View("AzureAISearchIndexSettings_Fields_SummaryAdmin", settings).Location("Content:1"),
+            View("AzureAISearchIndexSettings_Buttons_SummaryAdmin", settings).Location("Actions:5"),
+            View("AzureAISearchIndexSettings_DefaultTags_SummaryAdmin", settings).Location("Tags:5")
+        );
+    }
+
+    public override IDisplayResult Edit(AzureAISearchIndexSettings settings, BuildEditorContext context)
+    {
+        return Initialize<AzureAISettingsViewModel>("AzureAISearchIndexSettingsFields_Edit", model =>
+        {
+            model.AnalyzerName = settings.AnalyzerName ?? AzureAISearchDefaultOptions.DefaultAnalyzer;
+            model.IndexName = settings.IndexName;
+            model.IsNew = context.IsNew;
+            model.Analyzers = _azureAIOptions.Analyzers.Select(x => new SelectListItem(x, x));
+        }).Location("Content:1");
+    }
+
+    public override async Task<IDisplayResult> UpdateAsync(AzureAISearchIndexSettings settings, UpdateEditorContext context)
+    {
+        var model = new AzureAISettingsViewModel();
+
+        await context.Updater.TryUpdateModelAsync(model, Prefix);
+
+        if (context.IsNew)
+        {
+            if (string.IsNullOrWhiteSpace(model.IndexName))
+            {
+                context.Updater.ModelState.AddModelError(Prefix, nameof(model.IndexName), S["The index name is required."]);
+            }
+            else if (!AzureAISearchIndexNamingHelper.TryGetSafeIndexName(model.IndexName, out var indexName) || indexName != model.IndexName)
+            {
+                context.Updater.ModelState.AddModelError(Prefix, nameof(model.IndexName), S["The index name contains forbidden characters."]);
+            }
+            else if (await _indexManager.ExistsAsync(model.IndexName))
+            {
+                context.Updater.ModelState.AddModelError(Prefix, nameof(AzureAISettingsViewModel.IndexName), S["An index named <em>{0}</em> already exist in Azure AI Search server.", model.IndexName]);
+            }
+
+            settings.IndexName = model.IndexName;
+            settings.IndexFullName = _indexManager.GetFullIndexName(model.IndexName);
+        }
+
+        settings.AnalyzerName = model.AnalyzerName;
+        settings.QueryAnalyzerName = model.AnalyzerName;
+
+        if (string.IsNullOrEmpty(settings.AnalyzerName))
+        {
+            settings.AnalyzerName = AzureAISearchDefaultOptions.DefaultAnalyzer;
+        }
+
+        if (string.IsNullOrEmpty(settings.QueryAnalyzerName))
+        {
+            settings.QueryAnalyzerName = settings.AnalyzerName;
+        }
+
+        return Edit(settings, context);
+    }
+}

src/OrchardCore.Modules/OrchardCore.Search.AzureAI/Drivers/ContentAzureAISearchIndexSettingsDisplayDriver.cs

@@ -0,0 +1,68 @@
+using System.Globalization;
+using Microsoft.AspNetCore.Mvc.Rendering;
+using Microsoft.Extensions.Localization;
+using OrchardCore.DisplayManagement.Handlers;
+using OrchardCore.DisplayManagement.Views;
+using OrchardCore.Entities;
+using OrchardCore.Mvc.ModelBinding;
+using OrchardCore.Search.AzureAI.Models;
+using OrchardCore.Search.AzureAI.ViewModels;
+
+namespace OrchardCore.Search.AzureAI.Drivers;
+
+internal sealed class ContentAzureAISearchIndexSettingsDisplayDriver : DisplayDriver<AzureAISearchIndexSettings>
+{
+    internal readonly IStringLocalizer S;
+
+    public ContentAzureAISearchIndexSettingsDisplayDriver(IStringLocalizer<ContentAzureAISearchIndexSettingsDisplayDriver> stringLocalizer)
+    {
+        S = stringLocalizer;
+    }
+
+    public override IDisplayResult Edit(AzureAISearchIndexSettings settings, BuildEditorContext context)
+    {
+        if (!string.Equals(AzureAISearchConstants.ContentsIndexSource, settings.Source, StringComparison.OrdinalIgnoreCase))
+        {
+            return null;
+        }
+
+        return Initialize<ContentIndexMetadataViewModel>("ContentIndexMetadata_Edit", model =>
+        {
+            var metadata = settings.As<ContentIndexMetadata>();
+
+            model.IndexLatest = metadata.IndexLatest;
+            model.IndexedContentTypes = metadata.IndexedContentTypes;
+            model.Culture = metadata.Culture;
+
+            model.Cultures = CultureInfo.GetCultures(CultureTypes.AllCultures)
+            .Select(x => new SelectListItem { Text = $"{x.Name} ({x.DisplayName})", Value = x.Name });
+
+        }).Location("Content:5");
+    }
+
+    public override async Task<IDisplayResult> UpdateAsync(AzureAISearchIndexSettings settings, UpdateEditorContext context)
+    {
+        if (!string.Equals(AzureAISearchConstants.ContentsIndexSource, settings.Source, StringComparison.OrdinalIgnoreCase))
+        {
+            return null;
+        }
+
+        var model = new ContentIndexMetadataViewModel();
+
+        await context.Updater.TryUpdateModelAsync(model, Prefix);
+
+        if (model.IndexedContentTypes is null || model.IndexedContentTypes.Length == 0)
+        {
+            context.Updater.ModelState.AddModelError(Prefix, nameof(model.IndexedContentTypes), S["At least one content type must be selected."]);
+        }
+
+        settings.Alter<ContentIndexMetadata>(m =>
+        {
+            m.IndexLatest = model.IndexLatest;
+            m.IndexedContentTypes = model.IndexedContentTypes ?? [];
+            m.Culture = model.Culture;
+        });
+
+        return Edit(settings, context);
+    }
+}

src/OrchardCore.Modules/OrchardCore.Search.AzureAI/Migrations/AzureAISearchIndexSettingsMigrations.cs

@@ -0,0 +1,138 @@
+using System.Text.Json.Nodes;
+using Dapper;
+using Microsoft.Extensions.DependencyInjection;
+using OrchardCore.Data;
+using OrchardCore.Data.Migration;
+using OrchardCore.Documents;
+using OrchardCore.Entities;
+using OrchardCore.Environment.Shell.Scope;
+using OrchardCore.Search.AzureAI.Models;
+using YesSql;
+using YesSql.Sql;
+
+namespace OrchardCore.Search.AzureAI.Migrations;
+
+/// <summary>
+/// In version 3, we introduced Source, Id and the ability to add metadata to index settings.
+/// This migration will migrate any index that was created before v3 to use the new structure.
+/// This migration will be removed in future releases.
+/// </summary>
+internal sealed class AzureAISearchIndexSettingsMigrations : DataMigration
+{
+#pragma warning disable CA1822 // Mark members as static
+    public int Create()
+#pragma warning restore CA1822 // Mark members as static
+    {
+        ShellScope.AddDeferredTask(async scope =>
+        {
+            var store = scope.ServiceProvider.GetRequiredService<IStore>();
+            var dbConnectionAccessor = scope.ServiceProvider.GetRequiredService<IDbConnectionAccessor>();
+            var settingsManager = scope.ServiceProvider.GetRequiredService<IDocumentManager<AzureAISearchIndexSettingsDocument>>();
+
+            var documentTableName = store.Configuration.TableNameConvention.GetDocumentTable();
+            var table = $"{store.Configuration.TablePrefix}{documentTableName}";
+            var dialect = store.Configuration.SqlDialect;
+            var quotedTableName = dialect.QuoteForTableName(table, store.Configuration.Schema);
+            var quotedContentColumnName = dialect.QuoteForColumnName("Content");
+            var quotedTypeColumnName = dialect.QuoteForColumnName("Type");
+
+            var sqlBuilder = new SqlBuilder(store.Configuration.TablePrefix, store.Configuration.SqlDialect);
+            sqlBuilder.AddSelector(quotedContentColumnName);
+            sqlBuilder.From(quotedTableName);
+            sqlBuilder.WhereAnd($" {quotedTypeColumnName} = 'OrchardCore.Search.AzureAI.Models.AzureAISearchIndexSettingsDocument, OrchardCore.Search.AzureAI.Core' ");
+            sqlBuilder.Take("1");
+
+            await using var connection = dbConnectionAccessor.CreateConnection();
+            await connection.OpenAsync();
+            var jsonContent = await connection.QueryFirstOrDefaultAsync<string>(sqlBuilder.ToSqlString());
+
+            if (string.IsNullOrEmpty(jsonContent))
+            {
+                return;
+            }
+
+            var jsonObject = JsonNode.Parse(jsonContent);
+
+            if (jsonObject["IndexSettings"] is not JsonObject indexesObject)
+            {
+                return;
+            }
+
+            var document = await settingsManager.GetOrCreateMutableAsync();
+
+            foreach (var indexObject in indexesObject)
+            {
+                var source = indexObject.Value["Source"]?.GetValue<string>();
+
+                if (!string.IsNullOrEmpty(source))
+                {
+                    // No migration is needed.
+                    continue;
+                }
+
+                var indexName = indexObject.Value["IndexName"]?.GetValue<string>();
+
+                if (string.IsNullOrEmpty(indexName))
+                {
+                    // Bad index! this is a scenario that should never happen.
+                    continue;
+                }
+
+                if (!document.IndexSettings.TryGetValue(indexName, out var indexSettings))
+                {
+                    // Bad index! this is a scenario that should never happen.
+                    continue;
+                }
+
+                indexSettings.Source = AzureAISearchConstants.ContentsIndexSource;
+
+                if (string.IsNullOrEmpty(indexSettings.Id))
+                {
+                    indexSettings.Id = IdGenerator.GenerateId();
+                }
+
+                var metadata = indexSettings.As<ContentIndexMetadata>();
+
+                if (string.IsNullOrEmpty(metadata.Culture))
+                {
+                    metadata.Culture = indexObject.Value[nameof(ContentIndexMetadata.Culture)]?.GetValue<string>();
+                }
+
+                var indexLatest = indexObject.Value[nameof(ContentIndexMetadata.IndexLatest)]?.GetValue<bool>();
+
+                if (indexLatest.HasValue)
+                {
+                    metadata.IndexLatest = indexLatest.Value;
+                }
+
+                var indexContentTypes = indexObject.Value[nameof(ContentIndexMetadata.IndexedContentTypes)]?.AsArray();
+
+                if (indexContentTypes is not null)
+                {
+                    var items = new HashSet<string>();
+
+                    foreach (var indexContentType in indexContentTypes)
+                    {
+                        var value = indexContentType.GetValue<string>();
+
+                        if (!string.IsNullOrEmpty(value))
+                        {
+                            items.Add(value);
+                        }
+                    }
+
+                    metadata.IndexedContentTypes = items.ToArray();
+                }
+
+                indexSettings.Put(metadata);
+
+                document.IndexSettings.Remove(indexName);
+                document.IndexSettings[indexSettings.Id] = indexSettings;
+            }
+
+            await settingsManager.UpdateAsync(document);
+        });
+
+        return 1;
+    }
+}

src/OrchardCore.Modules/OrchardCore.Search.AzureAI/Startup.cs

@@ -1,6 +1,8 @@
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Localization;
 using OrchardCore.ContentTypes.Editors;
+using OrchardCore.Data.Migration;
 using OrchardCore.Deployment;
 using OrchardCore.DisplayManagement.Handlers;
 using OrchardCore.Modules;
@@ -9,6 +11,8 @@
 using OrchardCore.Search.AzureAI.Deployment;
 using OrchardCore.Search.AzureAI.Drivers;
 using OrchardCore.Search.AzureAI.Handlers;
+using OrchardCore.Search.AzureAI.Migrations;
+using OrchardCore.Search.AzureAI.Models;
 using OrchardCore.Search.AzureAI.Services;
 
 namespace OrchardCore.Search.AzureAI;
@@ -20,6 +24,11 @@ public override void ConfigureServices(IServiceCollection services)
         services.AddAzureAISearchServices();
         services.AddSiteDisplayDriver<AzureAISearchDefaultSettingsDisplayDriver>();
         services.AddNavigationProvider<AdminMenu>();
+
+        services.AddDisplayDriver<AzureAISearchIndexSettings, AzureAISearchIndexSettingsDisplayDriver>();
+        services.AddScoped<IAzureAISearchIndexSettingsHandler, AzureAISearchIndexHandler>();
+
+        services.AddDataMigration<AzureAISearchIndexSettingsMigrations>();
     }
 }
 
@@ -45,6 +54,32 @@ public override void ConfigureServices(IServiceCollection services)
     }
 }
 
+[RequireFeatures("OrchardCore.Contents")]
+public sealed class ContentsStartup : StartupBase
+{
+    private readonly IStringLocalizer S;
+
+    public ContentsStartup(IStringLocalizer<ContentsStartup> stringLocalizer)
+    {
+        S = stringLocalizer;
+    }
+
+    public override void ConfigureServices(IServiceCollection services)
+    {
+        services.AddScoped<IAzureAISearchEvents, ContentAzureAISearchEvents>();
+        services.AddDisplayDriver<AzureAISearchIndexSettings, ContentAzureAISearchIndexSettingsDisplayDriver>();
+        services.AddScoped<IAzureAISearchIndexSettingsHandler, ContentAzureAISearchIndexHandler>();
+        services.Configure<AzureAISearchOptions>(options =>
+        {
+            options.AddIndexSource(AzureAISearchConstants.ContentsIndexSource, o =>
+            {
+                o.DisplayName = S["Contents"];
+                o.Description = S["Create an index based on content items."];
+            });
+        });
+    }
+}
+
 [RequireFeatures("OrchardCore.Deployment")]
 public sealed class DeploymentStartup : StartupBase
 {

src/OrchardCore.Modules/OrchardCore.Search.AzureAI/ViewModels/AdminIndexViewModel.cs

@@ -5,10 +5,12 @@ namespace OrchardCore.Search.AzureAI.ViewModels;
 public class AdminIndexViewModel
 {
     [BindNever]
-    public IEnumerable<IndexViewModel> Indexes { get; set; }
+    public IList<AzureAIIndexEntry> Indexes { get; set; }
 
     public AzureAIIndexOptions Options { get; set; } = new();
 
     [BindNever]
     public dynamic Pager { get; set; }
+
+    public IEnumerable<string> SourceNames { get; set; }
 }

src/OrchardCore.Modules/OrchardCore.Search.AzureAI/ViewModels/AzureAIIndexEntry.cs

@@ -0,0 +1,11 @@
+using OrchardCore.DisplayManagement;
+using OrchardCore.Search.AzureAI.Models;
+
+namespace OrchardCore.Search.AzureAI.ViewModels;
+
+public class AzureAIIndexEntry
+{
+    public AzureAISearchIndexSettings Index { get; set; }
+
+    public IShape Shape { get; set; }
+}