Skip to content

Commit ada5b83

Browse files
authored
.NET: Add rag samples with sample TextSearchStore (#1664)
* Port store for adding text to a vector store to AF * Fix typo. * Change TextSearchStore to sample, and add sample to use it and do rag with a custom schema * Add more tests and fix broken ones * Fix merge issue * Fix sample after merge. * Convert TextSearchStore to use Dynamic mode to be AOT compatible. * Add some more clarification on when to use assistant messages in rag searches.
1 parent 64fc3f3 commit ada5b83

File tree

18 files changed

+1079
-42
lines changed

18 files changed

+1079
-42
lines changed

dotnet/Directory.Packages.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,11 @@
6565
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.10" />
6666
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="9.0.10" />
6767
<PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="$(AspireAppHostSdkVersion)" />
68+
<PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="9.7.0" />
6869
<!-- Vector Stores -->
6970
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.66.0" />
7071
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.InMemory" Version="1.66.0-preview" />
72+
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.66.0-preview" />
7173
<PackageVersion Include="Microsoft.SemanticKernel.Agents.Core" Version="1.66.0" />
7274
<PackageVersion Include="Microsoft.SemanticKernel.Agents.OpenAI" Version="1.66.0-preview" />
7375
<PackageVersion Include="Microsoft.SemanticKernel.Agents.AzureAI" Version="1.66.0-preview" />

dotnet/agent-framework-dotnet.slnx

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@
6767
<Project Path="samples/GettingStarted/AgentWithOpenAI/Agent_OpenAI_Step01_Running/Agent_OpenAI_Step01_Running.csproj" />
6868
<Project Path="samples/GettingStarted/AgentWithOpenAI/Agent_OpenAI_Step02_Reasoning/Agent_OpenAI_Step02_Reasoning.csproj" />
6969
</Folder>
70+
<Folder Name="/Samples/GettingStarted/AgentWithRAG/">
71+
<File Path="samples/GettingStarted/AgentWithRAG/README.md" />
72+
<Project Path="samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/AgentWithRAG_Step01_BasicTextRAG.csproj" />
73+
<Project Path="samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/AgentWithRAG_Step02_ExternalDataSourceRAG.csproj" />
74+
</Folder>
7075
<Folder Name="/Samples/GettingStarted/ModelContextProtocol/">
7176
<File Path="samples/GettingStarted/ModelContextProtocol/README.md" />
7277
<Project Path="samples/GettingStarted/ModelContextProtocol/Agent_MCP_Server/Agent_MCP_Server.csproj" />

dotnet/samples/Catalog/AgentWithTextSearchRag/Program.cs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,29 +52,29 @@
5252
{
5353
results.Add(new()
5454
{
55-
Name = "Contoso Outdoors Return Policy",
56-
Link = "https://contoso.com/policies/returns",
57-
Value = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
55+
SourceName = "Contoso Outdoors Return Policy",
56+
SourceLink = "https://contoso.com/policies/returns",
57+
Text = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
5858
});
5959
}
6060

6161
if (query.Contains("shipping", StringComparison.OrdinalIgnoreCase))
6262
{
6363
results.Add(new()
6464
{
65-
Name = "Contoso Outdoors Shipping Guide",
66-
Link = "https://contoso.com/help/shipping",
67-
Value = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
65+
SourceName = "Contoso Outdoors Shipping Guide",
66+
SourceLink = "https://contoso.com/help/shipping",
67+
Text = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
6868
});
6969
}
7070

7171
if (query.Contains("tent", StringComparison.OrdinalIgnoreCase) || query.Contains("fabric", StringComparison.OrdinalIgnoreCase))
7272
{
7373
results.Add(new()
7474
{
75-
Name = "TrailRunner Tent Care Instructions",
76-
Link = "https://contoso.com/manuals/trailrunner-tent",
77-
Value = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
75+
SourceName = "TrailRunner Tent Care Instructions",
76+
SourceLink = "https://contoso.com/manuals/trailrunner-tent",
77+
Text = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
7878
});
7979
}
8080

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net9.0</TargetFramework>
6+
7+
<Nullable>enable</Nullable>
8+
<ImplicitUsings>enable</ImplicitUsings>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="Azure.AI.OpenAI" />
13+
<PackageReference Include="Azure.Identity" />
14+
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
15+
<PackageReference Include="Microsoft.SemanticKernel.Connectors.InMemory" />
16+
</ItemGroup>
17+
18+
<ItemGroup>
19+
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
20+
</ItemGroup>
21+
22+
</Project>
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
// This sample shows how to use TextSearchProvider to add retrieval augmented generation (RAG) capabilities to an AI agent.
4+
// The sample uses an In-Memory vector store, which can easily be replaced with any other vector store that implements the Microsoft.Extensions.VectorData abstractions.
5+
// The TextSearchProvider runs a search against the vector store via the TextSearchStore before each model invocation and injects the results into the model context.
6+
// The TextSearchStore is a sample store implementation that hardcodes a storage schema and uses the vector store to store and retrieve documents.
7+
8+
using Azure.AI.OpenAI;
9+
using Azure.Identity;
10+
using Microsoft.Agents.AI;
11+
using Microsoft.Agents.AI.Data;
12+
using Microsoft.Agents.AI.Samples;
13+
using Microsoft.Extensions.AI;
14+
using Microsoft.Extensions.VectorData;
15+
using Microsoft.SemanticKernel.Connectors.InMemory;
16+
using OpenAI;
17+
18+
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
19+
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
20+
var embeddingDeploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME") ?? "text-embedding-3-large";
21+
22+
AzureOpenAIClient azureOpenAIClient = new(
23+
new Uri(endpoint),
24+
new AzureCliCredential());
25+
26+
// Create an In-Memory vector store that uses the Azure OpenAI embedding model to generate embeddings.
27+
VectorStore vectorStore = new InMemoryVectorStore(new()
28+
{
29+
EmbeddingGenerator = azureOpenAIClient.GetEmbeddingClient(embeddingDeploymentName).AsIEmbeddingGenerator()
30+
});
31+
32+
// Create a store that defines a storage schema, and uses the vector store to store and retrieve documents.
33+
TextSearchStore textSearchStore = new(vectorStore, "product-and-policy-info", 3072);
34+
35+
// Upload sample documents into the store.
36+
await textSearchStore.UpsertDocumentsAsync(GetSampleDocuments());
37+
38+
// Create an adapter function that the TextSearchProvider can use to run searches against the TextSearchStore.
39+
Func<string, CancellationToken, Task<IEnumerable<TextSearchProvider.TextSearchResult>>> SearchAdapter = async (text, ct) =>
40+
{
41+
// Here we are limiting the search results to the single top result to demonstrate that we are accurately matching
42+
// specific search results for each question, but in a real world case, more results should be used.
43+
var searchResults = await textSearchStore.SearchAsync(text, 1, ct);
44+
return searchResults.Select(r => new TextSearchProvider.TextSearchResult
45+
{
46+
SourceName = r.SourceName,
47+
SourceLink = r.SourceLink,
48+
Text = r.Text ?? string.Empty,
49+
RawRepresentation = r
50+
});
51+
};
52+
53+
// Configure the options for the TextSearchProvider.
54+
TextSearchProviderOptions textSearchOptions = new()
55+
{
56+
// Run the search prior to every model invocation.
57+
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
58+
};
59+
60+
// Create the AI agent with the TextSearchProvider as the AI context provider.
61+
AIAgent agent = azureOpenAIClient
62+
.GetChatClient(deploymentName)
63+
.CreateAIAgent(new ChatClientAgentOptions
64+
{
65+
Instructions = "You are a helpful support specialist for Contoso Outdoors. Answer questions using the provided context and cite the source document when available.",
66+
AIContextProviderFactory = ctx => ctx.SerializedState.ValueKind is not System.Text.Json.JsonValueKind.Null and not System.Text.Json.JsonValueKind.Undefined
67+
? new TextSearchProvider(SearchAdapter, ctx.SerializedState, ctx.JsonSerializerOptions, textSearchOptions)
68+
: new TextSearchProvider(SearchAdapter, textSearchOptions)
69+
});
70+
71+
AgentThread thread = agent.GetNewThread();
72+
73+
Console.WriteLine(">> Asking about returns\n");
74+
Console.WriteLine(await agent.RunAsync("Hi! I need help understanding the return policy.", thread));
75+
76+
Console.WriteLine("\n>> Asking about shipping\n");
77+
Console.WriteLine(await agent.RunAsync("How long does standard shipping usually take?", thread));
78+
79+
Console.WriteLine("\n>> Asking about product care\n");
80+
Console.WriteLine(await agent.RunAsync("What is the best way to maintain the TrailRunner tent fabric?", thread));
81+
82+
// Produces some sample search documents.
83+
// Each one contains a source name and link, which the agent can use to cite sources in its responses.
84+
static IEnumerable<TextSearchDocument> GetSampleDocuments()
85+
{
86+
yield return new TextSearchDocument
87+
{
88+
SourceId = "return-policy-001",
89+
SourceName = "Contoso Outdoors Return Policy",
90+
SourceLink = "https://contoso.com/policies/returns",
91+
Text = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
92+
};
93+
yield return new TextSearchDocument
94+
{
95+
SourceId = "shipping-guide-001",
96+
SourceName = "Contoso Outdoors Shipping Guide",
97+
SourceLink = "https://contoso.com/help/shipping",
98+
Text = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
99+
};
100+
yield return new TextSearchDocument
101+
{
102+
SourceId = "tent-care-001",
103+
SourceName = "TrailRunner Tent Care Instructions",
104+
SourceLink = "https://contoso.com/manuals/trailrunner-tent",
105+
Text = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
106+
};
107+
}
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
namespace Microsoft.Agents.AI.Samples;
4+
5+
/// <summary>
6+
/// Represents a document that can be used for Retrieval Augmented Generation (RAG) that stores textual data.
7+
/// </summary>
8+
public sealed class TextSearchDocument
9+
{
10+
/// <summary>
11+
/// Gets or sets an optional list of namespaces that the document should belong to.
12+
/// </summary>
13+
/// <remarks>
14+
/// A namespace is a logical grouping of documents, e.g. may include a group id to scope the document to a specific group of users.
15+
/// </remarks>
16+
public IList<string> Namespaces { get; set; } = [];
17+
18+
/// <summary>
19+
/// Gets or sets the content as text.
20+
/// </summary>
21+
public string? Text { get; set; }
22+
23+
/// <summary>
24+
/// Gets or sets an optional source ID for the document.
25+
/// </summary>
26+
/// <remarks>
27+
/// This ID should be unique within the collection that the document is stored in, and can
28+
/// be used to map back to the source artifact for this document.
29+
/// If updates need to be made later or the source document was deleted and this document
30+
/// also needs to be deleted, this id can be used to find the document again.
31+
/// </remarks>
32+
public string? SourceId { get; set; }
33+
34+
/// <summary>
35+
/// Gets or sets an optional name for the source document.
36+
/// </summary>
37+
/// <remarks>
38+
/// This can be used to provide display names for citation links when the document is referenced as
39+
/// part of a response to a query.
40+
/// </remarks>
41+
public string? SourceName { get; set; }
42+
43+
/// <summary>
44+
/// Gets or sets an optional link back to the source of the document.
45+
/// </summary>
46+
/// <remarks>
47+
/// This can be used to provide citation links when the document is referenced as
48+
/// part of a response to a query.
49+
/// </remarks>
50+
public string? SourceLink { get; set; }
51+
}

0 commit comments

Comments
 (0)