Skip to content

Commit 7ae2c77

Browse files
jamesmcroftjamesmcroft
committed
Add .NET sample for native PII document redaction using Azure AI Language
1 parent 3283d77 commit 7ae2c77

File tree

2 files changed

+768
-0
lines changed

2 files changed

+768
-0
lines changed

‎samples/dotnet/modules/samples/language/LanguageNativePIIClient.csx

+238
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
using System.Net.Http.Headers;
2+
using System.Text;
3+
using System.Text.Json;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
using Azure.Core;
7+
using Azure.Core.Pipeline;
8+
9+
public class LanguageNativePIIClient
10+
{
11+
private const string DefaultAuthorizationScope = "https://cognitiveservices.azure.com/.default";
12+
private readonly Uri _endpoint;
13+
private readonly HttpPipeline _pipeline;
14+
private readonly string _apiVersion;
15+
private readonly TokenCredential _credential;
16+
17+
public LanguageNativePIIClient(Uri endpoint, TokenCredential credential)
18+
{
19+
ArgumentNullException.ThrowIfNull(endpoint, nameof(endpoint));
20+
ArgumentNullException.ThrowIfNull(credential, nameof(credential));
21+
22+
_credential = credential;
23+
_pipeline = HttpPipelineBuilder.Build(new HttpPipelineOptions(ClientOptions.Default)
24+
{
25+
PerRetryPolicies = { new BearerTokenAuthenticationPolicy(credential, DefaultAuthorizationScope) }
26+
});
27+
_endpoint = endpoint;
28+
_apiVersion = "2024-11-15-preview";
29+
}
30+
31+
public virtual async Task<AnalyzeNativePIIResult> AnalyzeDocumentAsync(
32+
AnalyzeNativePIIOptions options,
33+
CancellationToken cancellationToken = default)
34+
{
35+
var accessToken = await _credential.GetTokenAsync(new TokenRequestContext([DefaultAuthorizationScope]), cancellationToken);
36+
37+
var requestContent = JsonSerializer.Serialize(options, JsonSerializerOptions.Web);
38+
39+
var request = _pipeline.CreateRequest();
40+
request.Content = RequestContent.Create(Encoding.UTF8.GetBytes(requestContent));
41+
request.Method = RequestMethod.Post;
42+
request.Uri.Reset(_endpoint);
43+
request.Uri.AppendPath("/language", escape: false);
44+
request.Uri.AppendPath("/analyze-documents/", escape: false);
45+
request.Uri.AppendPath("jobs", escape: false);
46+
request.Uri.AppendQuery("api-version", _apiVersion, true);
47+
request.Headers.Add("Authorization", new AuthenticationHeaderValue("Bearer", accessToken.Token).ToString());
48+
request.Headers.Add("Accept", "application/json");
49+
request.Headers.Add("Content-Type", "application/json");
50+
51+
var response = await _pipeline.SendRequestAsync(request, cancellationToken);
52+
53+
if (response.Status == 202)
54+
{
55+
response.Headers.TryGetValue("Operation-Location", out var operationLocation);
56+
57+
if (operationLocation != null)
58+
{
59+
var operationUri = new Uri(operationLocation);
60+
61+
while (true)
62+
{
63+
var operationRequest = _pipeline.CreateRequest();
64+
operationRequest.Method = RequestMethod.Get;
65+
operationRequest.Uri.Reset(operationUri);
66+
var operationResponse = await _pipeline.SendRequestAsync(operationRequest, cancellationToken);
67+
if (operationResponse.Status == 200)
68+
{
69+
var operationContent = operationResponse.Content.ToString();
70+
var result = JsonSerializer.Deserialize<AnalyzeNativePIIResult>(operationContent, JsonSerializerOptions.Web);
71+
if (result != null)
72+
{
73+
if (result.Errors != null && result.Errors.Any())
74+
{
75+
throw new InvalidOperationException(JsonSerializer.Serialize(result.Errors, JsonSerializerOptions.Web));
76+
}
77+
78+
var taskErrors = result.Tasks.Items.SelectMany(x => x.Results.Errors);
79+
if (taskErrors.Any())
80+
{
81+
throw new InvalidOperationException(JsonSerializer.Serialize(taskErrors, JsonSerializerOptions.Web));
82+
}
83+
84+
if (result.Status == "succeeded")
85+
{
86+
return result;
87+
}
88+
}
89+
else
90+
{
91+
throw new InvalidOperationException("No result returned.");
92+
}
93+
}
94+
95+
await Task.Delay(1000, cancellationToken);
96+
}
97+
}
98+
}
99+
100+
throw new InvalidOperationException($"Unexpected response: {response.Status}. {response.Content}");
101+
}
102+
}
103+
104+
public class AnalyzeNativePIIResult
105+
{
106+
public string JobId { get; set; }
107+
108+
public DateTime LastUpdatedDateTime { get; set; }
109+
110+
public DateTime CreatedDateTime { get; set; }
111+
112+
public DateTime ExpirationDateTime { get; set; }
113+
114+
public string Status { get; set; }
115+
116+
public IEnumerable<AnalyzeNativePIIResultError>? Errors { get; set; }
117+
118+
public string DisplayName { get; set; }
119+
120+
public AnalyzeNativePIIResultTasks Tasks { get; set; }
121+
}
122+
123+
public class AnalyzeNativePIIResultTasks
124+
{
125+
public int Completed { get; set; }
126+
127+
public int Failed { get; set; }
128+
129+
public int InProgress { get; set; }
130+
131+
public int Total { get; set; }
132+
133+
public IEnumerable<AnalyzeNativePIIResultTask> Items { get; set; }
134+
}
135+
136+
public class AnalyzeNativePIIResultTask
137+
{
138+
public string Kind { get; set; }
139+
140+
public string TaskName { get; set; }
141+
142+
public DateTime LastUpdateDateTime { get; set; }
143+
144+
public string Status { get; set; }
145+
146+
public AnalyzeNativePIIResultTaskResult Results { get; set; }
147+
}
148+
149+
public class AnalyzeNativePIIResultTaskResult
150+
{
151+
public IEnumerable<AnalyzeNativePIIResultDocument> Documents { get; set; }
152+
153+
public IEnumerable<AnalyzeNativePIIResultError>? Errors { get; set; }
154+
155+
public string ModelVersion { get; set; }
156+
}
157+
158+
public class AnalyzeNativePIIResultDocument
159+
{
160+
public string Id { get; set; }
161+
162+
public AnalyzeNativePIIResultDocumentLocation Source { get; set; }
163+
164+
public IEnumerable<AnalyzeNativePIIResultDocumentLocation> Targets { get; set; }
165+
166+
public IEnumerable<string>? Warnings { get; set; }
167+
}
168+
169+
public class AnalyzeNativePIIResultDocumentLocation
170+
{
171+
public string Kind { get; set; }
172+
173+
public string Location { get; set; }
174+
}
175+
176+
public class AnalyzeNativePIIResultError
177+
{
178+
public string Id { get; set; }
179+
180+
public Dictionary<string, object> Error { get; set; }
181+
}
182+
183+
public class AnalyzeNativePIIOptions(string displayName, AnalyzeInputOptions analysisInput, IEnumerable<AnalyzeNativePIIInputTask> tasks)
184+
{
185+
public string DisplayName { get; set; } = displayName;
186+
187+
public AnalyzeInputOptions AnalysisInput { get; set; } = analysisInput;
188+
189+
public IEnumerable<AnalyzeNativePIIInputTask> Tasks { get; set; } = tasks;
190+
}
191+
192+
public class AnalyzeNativePIIInputTask(string taskName, AnalyzeInputTaskParameters parameters)
193+
{
194+
public string Kind => "PiiEntityRecognition";
195+
196+
public string TaskName { get; set; } = taskName;
197+
198+
public AnalyzeInputTaskParameters Parameters { get; set; } = parameters;
199+
}
200+
201+
public class AnalyzeInputTaskParameters(AnalyzeInputTaskRedactionPolicy redactionPolicy, IEnumerable<string> piiCategories, bool excludeExtractionData)
202+
{
203+
public AnalyzeInputTaskRedactionPolicy RedactionPolicy { get; set; } = redactionPolicy;
204+
205+
public IEnumerable<string> PiiCategories { get; set; } = piiCategories;
206+
207+
public bool ExcludeExtractionData { get; set; } = excludeExtractionData;
208+
}
209+
210+
public class AnalyzeInputTaskRedactionPolicy(string policyKind = AnalyzeInputTaskRedactionPolicy.CharacterMask)
211+
{
212+
public const string NoMask = "noMask";
213+
public const string CharacterMask = "characterMask";
214+
public const string EntityMask = "entityMask";
215+
216+
public string PolicyKind { get; set; } = CharacterMask;
217+
}
218+
219+
public class AnalyzeInputOptions(IEnumerable<AnalyzeInputDocument> documents)
220+
{
221+
public IEnumerable<AnalyzeInputDocument> Documents { get; set; } = documents;
222+
}
223+
224+
public class AnalyzeInputDocument(string id, AnalyzeInputDocumentLocation source, AnalyzeInputDocumentLocation target, string language = "en")
225+
{
226+
public string Language { get; set; } = language;
227+
228+
public string Id { get; set; } = id;
229+
230+
public AnalyzeInputDocumentLocation Source { get; set; } = source;
231+
232+
public AnalyzeInputDocumentLocation Target { get; set; } = target;
233+
}
234+
235+
public class AnalyzeInputDocumentLocation(string location)
236+
{
237+
public string Location { get; set; } = location;
238+
}

0 commit comments

Comments
 (0)