Skip to content

Commit eb09f7a

Browse files
amankrxamankrx
committed
Add implementation for Azure Blob Store
1 parent 36d1c43 commit eb09f7a

File tree

10 files changed

+2135
-43
lines changed

10 files changed

+2135
-43
lines changed

‎Cargo.lock‎

Lines changed: 312 additions & 37 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎nativelink-config/examples/azure_blob_backend.json5‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
{
2+
"stores": [
3+
{
4+
"name": "CAS_MAIN_STORE",
5+
"verify": {
6+
"backend": {
7+
"dedup": {
8+
"index_store": {
9+
"fast_slow": {
10+
"fast": {
11+
"filesystem": {
12+
"content_path": "/tmp/nativelink/data/content_path-index",
13+
"temp_path": "/tmp/nativelink/data/tmp_path-index",
14+
"eviction_policy": {
15+
"max_bytes": 500000000 // 500mb
16+
}
17+
}
18+
},
19+
"slow": {
20+
"experimental_cloud_object_store": {
21+
"provider": "azure",
22+
"account_name": "storageaccamannativelink",
23+
"container": "storageacctaman",
24+
"key_prefix": "test-prefix-index/",
25+
"retry": {
26+
"max_retries": 6,
27+
"delay": 0.3,
28+
"jitter": 0.5
29+
},
30+
"multipart_max_concurrent_uploads": 10
31+
}
32+
}
33+
}
34+
},
35+
"content_store": {
36+
"compression": {
37+
"compression_algorithm": {
38+
"lz4": {}
39+
},
40+
"backend": {
41+
"fast_slow": {
42+
"fast": {
43+
"filesystem": {
44+
"content_path": "/tmp/nativelink/data/content_path-content",
45+
"temp_path": "/tmp/nativelink/data/tmp_path-content",
46+
"eviction_policy": {
47+
"max_bytes": 2000000000 // 2gb
48+
}
49+
}
50+
},
51+
"slow": {
52+
"experimental_cloud_object_store": {
53+
"provider": "azure",
54+
"account_name": "storageaccamannativelink",
55+
"container": "storageacctaman",
56+
"key_prefix": "test-prefix-dedup-cas/",
57+
"retry": {
58+
"max_retries": 6,
59+
"delay": 0.3,
60+
"jitter": 0.5
61+
},
62+
"multipart_max_concurrent_uploads": 10
63+
}
64+
}
65+
}
66+
}
67+
}
68+
}
69+
}
70+
},
71+
"verify_size": true
72+
}
73+
},
74+
{
75+
"name": "AC_MAIN_STORE",
76+
"fast_slow": {
77+
"fast": {
78+
"filesystem": {
79+
"content_path": "/tmp/nativelink/data/content_path-ac",
80+
"temp_path": "/tmp/nativelink/data/tmp_path-ac",
81+
"eviction_policy": {
82+
"max_bytes": 500000000 // 500mb
83+
}
84+
}
85+
},
86+
"slow": {
87+
"experimental_cloud_object_store": {
88+
"provider": "azure",
89+
"account_name": "storageaccamannativelink",
90+
"container": "storageacctaman",
91+
"key_prefix": "test-prefix-ac/",
92+
"retry": {
93+
"max_retries": 6,
94+
"delay": 0.3,
95+
"jitter": 0.5
96+
},
97+
"multipart_max_concurrent_uploads": 10
98+
}
99+
}
100+
}
101+
}
102+
],
103+
"schedulers": [
104+
{
105+
"name": "MAIN_SCHEDULER",
106+
"simple": {
107+
"supported_platform_properties": {
108+
"cpu_count": "minimum",
109+
"memory_kb": "minimum",
110+
"network_kbps": "minimum",
111+
"disk_read_iops": "minimum",
112+
"disk_read_bps": "minimum",
113+
"disk_write_iops": "minimum",
114+
"disk_write_bps": "minimum",
115+
"shm_size": "minimum",
116+
"gpu_count": "minimum",
117+
"gpu_model": "exact",
118+
"cpu_vendor": "exact",
119+
"cpu_arch": "exact",
120+
"cpu_model": "exact",
121+
"kernel_version": "exact",
122+
"docker_image": "priority",
123+
"lre-rs": "priority"
124+
}
125+
}
126+
}
127+
],
128+
"servers": [{
129+
"listener": {
130+
"http": {
131+
"socket_address": "0.0.0.0:50051"
132+
}
133+
},
134+
"services": {
135+
"cas": {
136+
"main": {
137+
"cas_store": "CAS_MAIN_STORE"
138+
}
139+
},
140+
"ac": {
141+
"main": {
142+
"ac_store": "AC_MAIN_STORE"
143+
}
144+
},
145+
"execution": {
146+
"main": {
147+
"cas_store": "CAS_MAIN_STORE",
148+
"scheduler": "MAIN_SCHEDULER"
149+
}
150+
},
151+
"capabilities": {
152+
"main": {
153+
"remote_execution": {
154+
"scheduler": "MAIN_SCHEDULER"
155+
}
156+
}
157+
},
158+
"bytestream": {
159+
"cas_stores": {
160+
"main": "CAS_MAIN_STORE"
161+
}
162+
},
163+
"health": {}
164+
}
165+
}]
166+
}

‎nativelink-config/src/stores.rs‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,26 @@ pub enum StoreSpec {
9999
/// "multipart_max_concurrent_uploads": 10
100100
/// }
101101
/// ```
102+
/// 3. **Azure Blob Store:**
103+
/// Azure Blob store will use Microsoft's Azure Blob service as a
104+
/// backend to store the files. This configuration can be used to
105+
/// share files across multiple instances.
102106
///
107+
/// **Example JSON Config:**
108+
/// ```json
109+
/// "experimental_cloud_object_store": {
110+
/// "provider": "azure",
111+
/// "account_name": "cloudshell1393657559",
112+
/// "container": "simple-test-container",
113+
/// "key_prefix": "folder/",
114+
/// "retry": {
115+
/// "max_retries": 6,
116+
/// "delay": 0.3,
117+
/// "jitter": 0.5
118+
/// },
119+
/// "max_concurrent_uploads": 10
120+
/// }
121+
/// ```
103122
ExperimentalCloudObjectStore(ExperimentalCloudObjectSpec),
104123

105124
/// Verify store is used to apply verifications to an underlying
@@ -751,6 +770,7 @@ pub struct EvictionPolicy {
751770
pub enum ExperimentalCloudObjectSpec {
752771
Aws(ExperimentalAwsSpec),
753772
Gcs(ExperimentalGcsSpec),
773+
Azure(ExperimentalAzureSpec),
754774
}
755775

756776
impl Default for ExperimentalCloudObjectSpec {
@@ -792,6 +812,22 @@ pub struct ExperimentalGcsSpec {
792812
pub common: CommonObjectSpec,
793813
}
794814

815+
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
816+
#[serde(deny_unknown_fields)]
817+
pub struct ExperimentalAzureSpec {
818+
/// The Azure Storage account name
819+
#[serde(default, deserialize_with = "convert_string_with_shellexpand")]
820+
pub account_name: String,
821+
822+
/// The container name to use as the backend
823+
#[serde(default, deserialize_with = "convert_string_with_shellexpand")]
824+
pub container: String,
825+
826+
/// Common retry and upload configuration
827+
#[serde(flatten)]
828+
pub common: CommonObjectSpec,
829+
}
830+
795831
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
796832
pub struct CommonObjectSpec {
797833
/// If you wish to prefix the location in the bucket. If None, no prefix will be used.
@@ -839,11 +875,17 @@ pub struct CommonObjectSpec {
839875
/// configuration will have http/1.1 and http/2 enabled for connection
840876
/// schemes. Http/2 should be disabled if environments have poor support
841877
/// or performance related to http/2. Safe to keep default unless
842-
/// underlying network environment, S3, or GCS API servers specify otherwise.
878+
/// underlying network environment or S3 API servers specify otherwise.
843879
///
844880
/// Default: false
845881
#[serde(default)]
846882
pub disable_http2: bool,
883+
884+
/// Service version to use when making requests.
885+
///
886+
/// Default: Latest supported version
887+
#[serde(default)]
888+
pub api_version: Option<String>,
847889
}
848890

849891
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]

‎nativelink-store/BUILD.bazel‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ rust_library(
1111
name = "nativelink-store",
1212
srcs = [
1313
"src/ac_utils.rs",
14+
"src/azure_blob_store.rs",
1415
"src/cas_utils.rs",
1516
"src/completeness_checking_store.rs",
1617
"src/compression_store.rs",
@@ -53,6 +54,9 @@ rust_library(
5354
"@crates//:aws-sdk-s3",
5455
"@crates//:aws-smithy-runtime-api",
5556
"@crates//:aws-smithy-types",
57+
"@crates//:azure_core",
58+
"@crates//:azure_storage",
59+
"@crates//:azure_storage_blobs",
5660
"@crates//:bincode",
5761
"@crates//:blake3",
5862
"@crates//:byteorder",
@@ -92,6 +96,7 @@ rust_test_suite(
9296
timeout = "short",
9397
srcs = [
9498
"tests/ac_utils_test.rs",
99+
"tests/azure_blob_store_test.rs",
95100
"tests/completeness_checking_store_test.rs",
96101
"tests/compression_store_test.rs",
97102
"tests/dedup_store_test.rs",
@@ -125,6 +130,9 @@ rust_test_suite(
125130
"@crates//:aws-smithy-runtime",
126131
"@crates//:aws-smithy-runtime-api",
127132
"@crates//:aws-smithy-types",
133+
"@crates//:azure_core",
134+
"@crates//:azure_storage",
135+
"@crates//:azure_storage_blobs",
128136
"@crates//:bincode",
129137
"@crates//:bytes",
130138
"@crates//:filetime",

‎nativelink-store/Cargo.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ aws-smithy-runtime-api = { version = "1.7.4", default-features = false, features
2626
aws-smithy-types = { version = "1.3.0", default-features = false, features = [
2727
"http-body-1-x",
2828
] }
29+
azure_core = { version = "0.21.0", default-features = false, features = [
30+
"hmac_rust",
31+
] }
32+
azure_storage = { version = "0.21.0", default-features = false, features = [
33+
"hmac_rust",
34+
] }
35+
azure_storage_blobs = { version = "0.21.0", default-features = false, features = [
36+
"hmac_rust",
37+
] }
2938
bincode = "1.3.3"
3039
blake3 = { version = "1.8.0", default-features = false }
3140
byteorder = { version = "1.5.0", default-features = false }

0 commit comments

Comments
 (0)