Skip to content

Commit 3b61cbf

Browse files
DaveCTurnerDaveCTurner
authored
Add support for Azure Managed Identity (elastic#111344)
With this commit, if no key or SAS token is supplied for an Azure repository then Elasticsearch will use the `DefaultAzureCredential` chain defined in the Azure SDK, which will obtain credentials from the instance metadata service when running on an Azure VM.
1 parent 72571df commit 3b61cbf

File tree

46 files changed

+2822
-20
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+2822
-20
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public class LicenseAnalyzer {
102102
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
103103
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
104104
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
105-
SOFTWARE\\.
105+
SOFTWARE\\.?
106106
""").replaceAll("\\s+", "\\\\s*"), Pattern.DOTALL)),
107107
new LicenseMatcher(
108108
"MIT-0",

‎docs/changelog/111344.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 111344
2+
summary: Add support for Azure Managed Identity
3+
area: Snapshot/Restore
4+
type: enhancement
5+
issues: []

‎gradle/verification-metadata.xml

+70
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,11 @@
129129
<sha256 value="e07002835364c67a9267fd019ee1019dc9ea865491b6b1d4f9b8c8809fc1bae7" origin="Generated by Gradle"/>
130130
</artifact>
131131
</component>
132+
<component group="com.azure" name="azure-identity" version="1.13.1">
133+
<artifact name="azure-identity-1.13.1.jar">
134+
<sha256 value="b479a7a1b6143212faaca3f4edc62d2929b14ea9aa48c80fe7cbe418928d6bcd" origin="Generated by Gradle"/>
135+
</artifact>
136+
</component>
132137
<component group="com.azure" name="azure-json" version="1.1.0">
133138
<artifact name="azure-json-1.1.0.jar">
134139
<sha256 value="114af9b1459c9c93190b2a82f427c0e93fbbb28896fc57f2585a9b821275ca56" origin="Generated by Gradle"/>
@@ -536,6 +541,11 @@
536541
<sha256 value="0cdd163ce3598a20fc04eee71b140b24f6f2a3b35f0a499dbbdd9852e83fbfaf" origin="Generated by Gradle"/>
537542
</artifact>
538543
</component>
544+
<component group="com.google.code.gson" name="gson" version="2.11.0">
545+
<artifact name="gson-2.11.0.jar">
546+
<sha256 value="57928d6e5a6edeb2abd3770a8f95ba44dce45f3b23b7a9dc2b309c581552a78b" origin="Generated by Gradle"/>
547+
</artifact>
548+
</component>
539549
<component group="com.google.code.gson" name="gson" version="2.2.4">
540550
<artifact name="gson-2.2.4.jar">
541551
<sha256 value="c0328cd07ca9e363a5acd00c1cf4afe8cf554bd6d373834981ba05cebec687fb" origin="Generated by Gradle"/>
@@ -546,6 +556,11 @@
546556
<sha256 value="c8fb4839054d280b3033f800d1f5a97de2f028eb8ba2eb458ad287e536f3f25f" origin="Generated by Gradle"/>
547557
</artifact>
548558
</component>
559+
<component group="com.google.crypto.tink" name="tink" version="1.14.0">
560+
<artifact name="tink-1.14.0.jar">
561+
<sha256 value="47b2248705e0c9771bc259f22465a79655c1296e2d47aaee852adb7cdacb6198" origin="Generated by Gradle"/>
562+
</artifact>
563+
</component>
549564
<component group="com.google.errorprone" name="error_prone_annotations" version="2.1.3">
550565
<artifact name="error_prone_annotations-2.1.3.jar">
551566
<sha256 value="03d0329547c13da9e17c634d1049ea2ead093925e290567e1a364fd6b1fc7ff8" origin="Generated by Gradle"/>
@@ -726,6 +741,11 @@
726741
<sha256 value="1b78b4a76a71512debfdff8f8fc5aef6bfd459f65758fecf7aff245e6e6301e4" origin="Generated by Gradle"/>
727742
</artifact>
728743
</component>
744+
<component group="com.google.protobuf" name="protobuf-java" version="4.27.0">
745+
<artifact name="protobuf-java-4.27.0.jar">
746+
<sha256 value="9072e60fe66cff5d6c0f11a1df21d8f3e4b29b5ee782b45c3fc75f59fbe2b839" origin="Generated by Gradle"/>
747+
</artifact>
748+
</component>
729749
<component group="com.google.protobuf" name="protobuf-java-util" version="3.21.9">
730750
<artifact name="protobuf-java-util-3.21.9.jar">
731751
<sha256 value="a78226d02551ca2cb34b24b4168a75b4a84041cc3333f3725bb5ad9806a80b77" origin="Generated by Gradle"/>
@@ -861,6 +881,16 @@
861881
<sha256 value="f1eae7a4e14944940e54605d7b512bdd3296efe6353af6bfb15c1b97ed06d8ec" origin="Generated by Gradle"/>
862882
</artifact>
863883
</component>
884+
<component group="com.microsoft.azure" name="msal4j" version="1.16.1">
885+
<artifact name="msal4j-1.16.1.jar">
886+
<sha256 value="871e50e97d61685d77480fc021b048eecb94d4a005b08f89e7f5d0a6a8ab1dc0" origin="Generated by Gradle"/>
887+
</artifact>
888+
</component>
889+
<component group="com.microsoft.azure" name="msal4j-persistence-extension" version="1.3.0">
890+
<artifact name="msal4j-persistence-extension-1.3.0.jar">
891+
<sha256 value="dfc41c817fbfa76057af6ffe4379dbca6a5e16b8e87df8bdda23f371756c2d09" origin="Generated by Gradle"/>
892+
</artifact>
893+
</component>
864894
<component group="com.microsoft.sqlserver" name="mssql-jdbc" version="6.2.1.jre7">
865895
<artifact name="mssql-jdbc-6.2.1.jre7.jar">
866896
<sha256 value="9cfa259450ae3471d2e6e2c3d8aefcce236e3daef8b3734d21dc93c3a5bbe806" origin="Generated by Gradle"/>
@@ -896,11 +926,21 @@
896926
<sha256 value="260647c459cc7de269b2a8f576e5693e434df888fd4801e92365fe18b9992cbc" origin="Generated by Gradle"/>
897927
</artifact>
898928
</component>
929+
<component group="com.nimbusds" name="content-type" version="2.3">
930+
<artifact name="content-type-2.3.jar">
931+
<sha256 value="60349793e006fba96b532cb0c21e10e969fe0db8d87f91c3b9eaf82ba2998895" origin="Generated by Gradle"/>
932+
</artifact>
933+
</component>
899934
<component group="com.nimbusds" name="lang-tag" version="1.4.4">
900935
<artifact name="lang-tag-1.4.4.jar">
901936
<sha256 value="e49d2c694bb80c7036c177f2aabf53b7156061a68bd19dfd60e2bd370709e0c5" origin="Generated by Gradle"/>
902937
</artifact>
903938
</component>
939+
<component group="com.nimbusds" name="lang-tag" version="1.7">
940+
<artifact name="lang-tag-1.7.jar">
941+
<sha256 value="e8c1c594e2425bdbea2d860de55c69b69fc5d59454452449a0f0913c2a5b8a31" origin="Generated by Gradle"/>
942+
</artifact>
943+
</component>
904944
<component group="com.nimbusds" name="nimbus-jose-jwt" version="4.41.1">
905945
<artifact name="nimbus-jose-jwt-4.41.1.jar">
906946
<sha256 value="fbfd0d5f2b2f86758b821daa5e79b5d7c965edd9dc1b2cc80b515df1c6ddc22d" origin="Generated by Gradle"/>
@@ -911,11 +951,21 @@
911951
<sha256 value="33ab8084fdae1d75be1b061b1489d4a12045bd7b50c2e24ff152911e4551ec07" origin="Generated by Gradle"/>
912952
</artifact>
913953
</component>
954+
<component group="com.nimbusds" name="nimbus-jose-jwt" version="9.37.3">
955+
<artifact name="nimbus-jose-jwt-9.37.3.jar">
956+
<sha256 value="12ae4a3a260095d7aeba2adea7ae396e8b9570db8b7b409e09a824c219cc0444" origin="Generated by Gradle"/>
957+
</artifact>
958+
</component>
914959
<component group="com.nimbusds" name="nimbus-jose-jwt" version="9.8.1">
915960
<artifact name="nimbus-jose-jwt-9.8.1.jar">
916961
<sha256 value="7664cf8c6f2adadf600287812b32878277beda54912eab9d4c2932cd50cb704a" origin="Generated by Gradle"/>
917962
</artifact>
918963
</component>
964+
<component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.9.1">
965+
<artifact name="oauth2-oidc-sdk-11.9.1.jar">
966+
<sha256 value="0820c9690966304d075347b88e81ae490213440fc4d2c84f3d370d41941b2b9c" origin="Generated by Gradle"/>
967+
</artifact>
968+
</component>
919969
<component group="com.nimbusds" name="oauth2-oidc-sdk" version="9.37">
920970
<artifact name="oauth2-oidc-sdk-9.37.jar">
921971
<sha256 value="44a04bbed5ae3f6d198aa73ee6b545c476e528ec1a267ef3e9f7033f886dd6fe" origin="Generated by Gradle"/>
@@ -1639,6 +1689,11 @@
16391689
<sha256 value="0271ae7fc162a5e69c337f36d86fdb94a8a232c5c42a80d8a7424071addd1fdc" origin="Generated by Gradle"/>
16401690
</artifact>
16411691
</component>
1692+
<component group="net.java.dev.jna" name="jna-platform" version="5.12.1">
1693+
<artifact name="jna-platform-5.12.1.jar">
1694+
<sha256 value="8ce969116cac95bd61b07a8d5e07174b352e63301473caac72c395e3c08488d2" origin="Generated by Gradle"/>
1695+
</artifact>
1696+
</component>
16421697
<component group="net.java.dev.jna" name="jna-platform" version="5.7.0">
16431698
<artifact name="jna-platform-5.7.0.jar">
16441699
<sha256 value="42e020705692eddbd285e2b72ef0ff468f51a926382569c45f4e9cea4602ad1e" origin="Generated by Gradle"/>
@@ -1664,6 +1719,11 @@
16641719
<sha256 value="0972bbc99437c4163acd09b630e6c77eab4cfab8a9594621c95466c0c6645396" origin="Generated by Gradle"/>
16651720
</artifact>
16661721
</component>
1722+
<component group="net.minidev" name="accessors-smart" version="2.5.0">
1723+
<artifact name="accessors-smart-2.5.0.jar">
1724+
<sha256 value="12314fc6881d66a413fd66370787adba16e504fbf7e138690b0f3952e3fbd321" origin="Generated by Gradle"/>
1725+
</artifact>
1726+
</component>
16671727
<component group="net.minidev" name="json-smart" version="2.3">
16681728
<artifact name="json-smart-2.3.jar">
16691729
<sha256 value="903f48c8aa4c3f6426440b8d32de89fa1dc23b1169abde25e4e1d068aa67708b" origin="Generated by Gradle"/>
@@ -1679,6 +1739,11 @@
16791739
<sha256 value="64072f56d9dff5040b2acec477c5d5e6bcebfc88c508f12acb26072d07942146" origin="Generated by Gradle"/>
16801740
</artifact>
16811741
</component>
1742+
<component group="net.minidev" name="json-smart" version="2.5.0">
1743+
<artifact name="json-smart-2.5.0.jar">
1744+
<sha256 value="432b9e545848c4141b80717b26e367f83bf33f19250a228ce75da6e967da2bc7" origin="Generated by Gradle"/>
1745+
</artifact>
1746+
</component>
16821747
<component group="net.nextencia" name="rrdiagram" version="0.9.4">
16831748
<artifact name="rrdiagram-0.9.4.jar">
16841749
<sha256 value="8f0855addca5320cfadedbf7d3d46b681f3f308b6e87d5d82f32637ba72256b6" origin="Generated by Gradle"/>
@@ -3312,6 +3377,11 @@
33123377
<sha256 value="c600d1ae61b5b0ff1391e00eb6fb390201e4612c3aaf2dc1b94050c8784840be" origin="Generated by Gradle"/>
33133378
</artifact>
33143379
</component>
3380+
<component group="org.cryptomator" name="siv-mode" version="1.5.2">
3381+
<artifact name="siv-mode-1.5.2.jar">
3382+
<sha256 value="81a33da58e5e878ffe8972ef93cd0bfc3a9daa0f5e3af4dd151be2b1d73018b4" origin="Generated by Gradle"/>
3383+
</artifact>
3384+
</component>
33153385
<component group="org.eclipse.jdt" name="ecj" version="3.33.0">
33163386
<artifact name="ecj-3.33.0.jar">
33173387
<sha256 value="f7686c4960cf70c2ebc5c500a73a8cfc04541b730c18f1c5c21329889b137f45" origin="Generated by Gradle"/>

‎modules/repository-azure/build.gradle

+115
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import org.apache.tools.ant.filters.ReplaceTokens
22
import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
3+
import org.elasticsearch.gradle.internal.test.RestIntegTestTask
34

45
/*
56
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -24,11 +25,14 @@ dependencies {
2425
// Microsoft
2526
api "com.azure:azure-core-http-netty:1.15.1"
2627
api "com.azure:azure-core:1.50.0"
28+
api "com.azure:azure-identity:1.13.1"
2729
api "com.azure:azure-json:1.1.0"
2830
api "com.azure:azure-storage-blob:12.26.1"
2931
api "com.azure:azure-storage-common:12.26.0"
3032
api "com.azure:azure-storage-internal-avro:12.11.1"
3133
api "com.azure:azure-xml:1.0.0"
34+
api "com.microsoft.azure:msal4j-persistence-extension:1.3.0"
35+
api "com.microsoft.azure:msal4j:1.16.1"
3236

3337
// Jackson
3438
api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
@@ -53,9 +57,24 @@ dependencies {
5357

5458
// Others
5559
api "com.fasterxml.woodstox:woodstox-core:6.4.0"
60+
api "com.github.stephenc.jcip:jcip-annotations:1.0-1"
61+
api "com.nimbusds:content-type:2.3"
62+
api "com.nimbusds:lang-tag:1.7"
63+
api "com.nimbusds:nimbus-jose-jwt:9.37.3"
64+
api "com.nimbusds:oauth2-oidc-sdk:11.9.1"
5665
api "jakarta.activation:jakarta.activation-api:1.2.1"
5766
api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3"
67+
api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
68+
api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
69+
api "net.minidev:accessors-smart:2.5.0"
70+
api "net.minidev:json-smart:2.5.0"
5871
api "org.codehaus.woodstox:stax2-api:4.2.1"
72+
api "org.ow2.asm:asm:9.3"
73+
74+
runtimeOnly "com.google.crypto.tink:tink:1.14.0"
75+
runtimeOnly "com.google.protobuf:protobuf-java:4.27.0"
76+
runtimeOnly "com.google.code.gson:gson:2.11.0"
77+
runtimeOnly "org.cryptomator:siv-mode:1.5.2"
5978

6079
implementation project(":modules:transport-netty4")
6180
implementation("org.slf4j:slf4j-api:${versions.slf4j}")
@@ -151,12 +170,94 @@ tasks.named("thirdPartyAudit").configure {
151170
'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
152171
// [missing classes] SLF4j includes an optional class that depends on an extension class. see Log4jLogger#createConverter
153172
// 'org.slf4j.ext.EventData' - bring back when https://github.com/elastic/elasticsearch/issues/93714 is done
173+
174+
// Optional dependency of tink
175+
'com.google.api.client.http.HttpHeaders',
176+
'com.google.api.client.http.HttpRequest',
177+
'com.google.api.client.http.HttpRequestFactory',
178+
'com.google.api.client.http.HttpResponse',
179+
'com.google.api.client.http.HttpTransport',
180+
'com.google.api.client.http.javanet.NetHttpTransport',
181+
'com.google.api.client.http.javanet.NetHttpTransport$Builder',
182+
183+
// Optional dependency of nimbus-jose-jwt and oauth2-oidc-sdk
184+
'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
185+
'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
186+
'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
187+
'org.bouncycastle.cert.X509CertificateHolder',
188+
'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
189+
'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
190+
'org.bouncycastle.crypto.InvalidCipherTextException',
191+
'org.bouncycastle.crypto.engines.AESEngine',
192+
'org.bouncycastle.crypto.modes.GCMBlockCipher',
193+
'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
194+
'org.bouncycastle.jce.provider.BouncyCastleProvider',
195+
'org.bouncycastle.openssl.PEMKeyPair',
196+
'org.bouncycastle.openssl.PEMParser',
197+
'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
198+
'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
199+
200+
// OAuth servlet support is optional and not required
201+
'jakarta.servlet.ServletRequest',
202+
'jakarta.servlet.http.HttpServletRequest',
203+
'jakarta.servlet.http.HttpServletResponse',
204+
'javax.servlet.ServletRequest',
205+
'javax.servlet.http.HttpServletRequest',
206+
'javax.servlet.http.HttpServletResponse',
207+
208+
// OpenSAML support is optional
209+
'org.joda.time.DateTime',
210+
'net.shibboleth.utilities.java.support.xml.SerializeSupport',
211+
'org.opensaml.core.config.InitializationException',
212+
'org.opensaml.core.config.InitializationService',
213+
'org.opensaml.core.xml.XMLObject',
214+
'org.opensaml.core.xml.XMLObjectBuilder',
215+
'org.opensaml.core.xml.XMLObjectBuilderFactory',
216+
'org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport',
217+
'org.opensaml.core.xml.io.Marshaller',
218+
'org.opensaml.core.xml.io.MarshallerFactory',
219+
'org.opensaml.core.xml.io.MarshallingException',
220+
'org.opensaml.core.xml.io.Unmarshaller',
221+
'org.opensaml.core.xml.io.UnmarshallerFactory',
222+
'org.opensaml.core.xml.schema.XSString',
223+
'org.opensaml.core.xml.schema.impl.XSStringBuilder',
224+
'org.opensaml.saml.saml2.core.Assertion',
225+
'org.opensaml.saml.saml2.core.Attribute',
226+
'org.opensaml.saml.saml2.core.AttributeStatement',
227+
'org.opensaml.saml.saml2.core.AttributeValue',
228+
'org.opensaml.saml.saml2.core.Audience',
229+
'org.opensaml.saml.saml2.core.AudienceRestriction',
230+
'org.opensaml.saml.saml2.core.AuthnContext',
231+
'org.opensaml.saml.saml2.core.AuthnContextClassRef',
232+
'org.opensaml.saml.saml2.core.AuthnStatement',
233+
'org.opensaml.saml.saml2.core.Conditions',
234+
'org.opensaml.saml.saml2.core.Issuer',
235+
'org.opensaml.saml.saml2.core.NameID',
236+
'org.opensaml.saml.saml2.core.Subject',
237+
'org.opensaml.saml.saml2.core.SubjectConfirmation',
238+
'org.opensaml.saml.saml2.core.SubjectConfirmationData',
239+
'org.opensaml.saml.security.impl.SAMLSignatureProfileValidator',
240+
'org.opensaml.security.credential.BasicCredential',
241+
'org.opensaml.security.credential.Credential',
242+
'org.opensaml.security.credential.UsageType',
243+
'org.opensaml.xmlsec.signature.Signature',
244+
'org.opensaml.xmlsec.signature.support.SignatureException',
245+
'org.opensaml.xmlsec.signature.support.SignatureValidator',
246+
'org.opensaml.xmlsec.signature.support.Signer',
154247
)
155248

156249
ignoreViolations(
157250
'javax.activation.MailcapCommandMap',
158251
'javax.activation.MimetypesFileTypeMap',
159252
'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException',
253+
254+
'com.google.protobuf.MessageSchema',
255+
'com.google.protobuf.UnsafeUtil',
256+
'com.google.protobuf.UnsafeUtil$1',
257+
'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
258+
'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
259+
'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
260+
'com.google.protobuf.UnsafeUtil$MemoryAccessor',
160261
)
161262
}
162263

@@ -189,6 +290,7 @@ tasks.named("processYamlRestTestResources") {
189290
tasks.named("internalClusterTest") {
190291
// this is tested explicitly in a separate test task
191292
exclude '**/AzureStorageCleanupThirdPartyTests.class'
293+
systemProperty "AZURE_POD_IDENTITY_AUTHORITY_HOST", "127.0.0.1:1" // ensure a fast failure
192294
}
193295

194296
tasks.named("yamlRestTest") {
@@ -199,6 +301,18 @@ tasks.named("yamlRestTest") {
199301
systemProperty 'test.azure.sas_token', azureSasToken
200302
}
201303

304+
tasks.register("managedIdentityYamlRestTest", RestIntegTestTask) {
305+
testClassesDirs = sourceSets.yamlRestTest.output.classesDirs
306+
classpath = sourceSets.yamlRestTest.runtimeClasspath
307+
}
308+
309+
tasks.named("managedIdentityYamlRestTest") {
310+
systemProperty 'test.azure.fixture', Boolean.toString(useFixture)
311+
systemProperty 'test.azure.account', azureAccount
312+
systemProperty 'test.azure.container', azureContainer
313+
// omitting key and sas_token so that we use a bearer token from the metadata service
314+
}
315+
202316
tasks.register("azureThirdPartyUnitTest", Test) {
203317
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
204318
SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
@@ -219,4 +333,5 @@ tasks.register('azureThirdPartyTest') {
219333

220334
tasks.named("check") {
221335
dependsOn("azureThirdPartyUnitTest")
336+
dependsOn("managedIdentityYamlRestTest")
222337
}

0 commit comments

Comments
 (0)