Skip to content

Commit 1df0a91

Browse files
jackowfishjackowfish
committed
Add S3 backend support to nydusify commit
Signed-off-by: Jack Decker <[email protected]>
1 parent 4909a90 commit 1df0a91

File tree

44 files changed

+2802
-230
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2802
-230
lines changed

‎.github/workflows/convert.yml‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ jobs:
108108
sudo docker run -d --restart=always -p 5000:5000 registry
109109
sudo mkdir convert-zran
110110
for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
111+
# skip influxdb which have some issue to convert use zran
112+
if [ "$I" = "influxdb" ]; then
113+
echo "skip influxdb image for zran conversion"
114+
continue
115+
fi
111116
echo "converting $I:latest to $I:nydus-nightly-oci-ref"
112117
ghcr_repo=${{ env.REGISTRY }}/${{ env.ORGANIZATION }}
113118
@@ -384,6 +389,15 @@ jobs:
384389
batchConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v6-batch/${I}.json) / 1000000")")
385390
echo "|${I}:latest|${zranConversionElapsed}|${v5ConversionElapsed}|${v6ConversionElapsed}|${batchConversionElapsed}|" >> $GITHUB_STEP_SUMMARY
386391
done
387-
- uses: geekyeggo/delete-artifact@v2
392+
- uses: geekyeggo/delete-artifact@v5
388393
with:
389-
name: '*'
394+
name: |
395+
nydusify-artifact
396+
nydus-artifact
397+
fsck-erofs-artifact
398+
convert-zran-metric
399+
convert-native-v5-metric
400+
convert-native-v6-metric
401+
convert-native-v6-batch-metric
402+
useGlob: false
403+
failOnError: false

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ nydus-static/
1212
.goreleaser.yml
1313
metadata.db
1414
tests/texture/zran/233c72f2b6b698c07021c4da367cfe2dff4f049efbaa885ca0ff760ea297865a
15+
vendor

‎APPROVERS.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Approvers
2+
3+
<!-- markdownlint-disable -->
4+
5+
| GitHub ID | Name | Email | Company |
6+
| :-------------------------------------------: | :-----------: | :-----------------------------: | :------------------------------------------: |
7+
| [adamqqqplay](https://github.com/adamqqqplay) | QinQi Zhai | [email protected] | Alibaba Group |
8+
| [power-more](https://github.com/power-more) | Shang Zhao | [email protected] | Alibaba Group |
9+
| [Desiki-high](https://github.com/Desiki-high) | Yadong Ding | [email protected] | Dalian University of Technology |
10+
| [Zephyr](https://github.com/Zephyrcf) | Changfu Zhang | [email protected] | University of Science and Technology Beijing |
11+
12+
<!-- markdownlint-restore -->

‎Cargo.lock‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,12 @@ path = "src/lib.rs"
3535
anyhow = "1"
3636
clap = { version = "4.0.18", features = ["derive", "cargo"] }
3737
flexi_logger = { version = "0.25", features = ["compress"] }
38-
fuse-backend-rs = "^0.12.0"
38+
fuse-backend-rs = { workspace = true }
3939
hex = "0.4.3"
4040
hyper = "0.14.11"
4141
hyperlocal = "0.8.0"
4242
lazy_static = "1"
43-
libc = "0.2"
43+
libc = "0.2.175"
4444
log = "0.4.8"
4545
log-panics = { version = "2.1.0", features = ["with-backtrace"] }
4646
mio = { version = "0.8", features = ["os-poll", "os-ext"] }
@@ -69,21 +69,21 @@ nydus-storage = { version = "0.7.0", path = "storage", features = [
6969
] }
7070
nydus-utils = { version = "0.5.0", path = "utils" }
7171

72-
vhost = { version = "0.11.0", features = ["vhost-user"], optional = true }
73-
vhost-user-backend = { version = "0.15.0", optional = true }
74-
virtio-bindings = { version = "0.1", features = [
72+
vhost = { workspace = true, features = ["vhost-user"], optional = true }
73+
vhost-user-backend = { workspace = true, optional = true }
74+
virtio-bindings = { workspace = true, features = [
7575
"virtio-v5_0_0",
7676
], optional = true }
77-
virtio-queue = { version = "0.12.0", optional = true }
78-
vm-memory = { version = "0.14.1", features = ["backend-mmap","backend-atomic"], optional = true }
79-
vmm-sys-util = { version = "0.12.1", optional = true }
77+
virtio-queue = { workspace = true, optional = true }
78+
vm-memory = { workspace = true, features = ["backend-mmap","backend-atomic"], optional = true }
79+
vmm-sys-util = { workspace = true, optional = true }
8080

8181
[build-dependencies]
8282
time = { version = "0.3.14", features = ["formatting"] }
8383

8484
[dev-dependencies]
8585
xattr = "1.0.1"
86-
vmm-sys-util = "0.12.1"
86+
vmm-sys-util = { workspace = true }
8787

8888
[features]
8989
default = [
@@ -128,3 +128,15 @@ members = [
128128
"upgrade",
129129
"utils",
130130
]
131+
132+
[workspace.dependencies]
133+
# rust-vmm crates
134+
vhost = "0.11.0"
135+
vhost-user-backend = "0.15.0"
136+
virtio-bindings = "0.1"
137+
virtio-queue = "0.12.0"
138+
vm-memory = "0.14.1"
139+
vmm-sys-util = "0.12.1"
140+
141+
# fuse-backend-rs which depends on rust-vmm
142+
fuse-backend-rs = "0.13.1"

‎MAINTAINERS.md‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@
22

33
<!-- markdownlint-disable -->
44

5-
| GitHub ID | Name | Email | Company |
6-
| :-------------------------------------------: | :---------: | :-----------------------------: | :-----------: |
7-
| [imeoer](https://github.com/imeoer) | Yan Song | [email protected] | Ant Group |
8-
| [bergwolf](https://github.com/bergwolf) | Peng Tao | [email protected] | Ant Group |
9-
| [jiangliu](https://github.com/jiangliu) | Jiang Liu | [email protected] | Alibaba Group |
10-
| [liubogithub](https://github.com/liubogithub) | Liu Bo | [email protected] | Alibaba Group |
11-
| [luodw](https://github.com/luodw) | daowen luo | [email protected] | ByteDance |
12-
| [changweige](https://github.com/changweige) | Changwei Ge | [email protected] | ByteDance |
13-
| [hsiangkao](https://github.com/hsiangkao) | Gao Xiang | [email protected] | Alibaba Group |
5+
| GitHub ID | Name | Email | Company |
6+
| :-------------------------------------------: | :-----------: | :--------------------------: | :-----------: |
7+
| [imeoer](https://github.com/imeoer) | Song Yan | [email protected] | Ant Group |
8+
| [bergwolf](https://github.com/bergwolf) | Tao Peng | [email protected] | Ant Group |
9+
| [jiangliu](https://github.com/jiangliu) | Jiang Liu | [email protected] | Alibaba Group |
10+
| [liubogithub](https://github.com/liubogithub) | Bo Liu | [email protected] | Alibaba Group |
11+
| [changweige](https://github.com/changweige) | Changwei Ge | [email protected] | ByteDance |
12+
| [hsiangkao](https://github.com/hsiangkao) | Xiang Gao | [email protected] | Alibaba Group |
13+
| [BraveY](https://github.com/BraveY) | Kaiyong Yang | [email protected] | Ant Group |
14+
| [liubin](https://github.com/liubin) | Bin Liu | [email protected] | Ant Group |
15+
| [NehemiahMi](https://github.com/NehemiahMi) | PengCheng Liu | [email protected] | Ant Group |
16+
| [gaius-qi](https://github.com/NehemiahMi) | Wenbo Qi | [email protected] | Ant Group |
1417

1518
<!-- markdownlint-restore -->

‎README.md‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,26 +48,26 @@ The following Benchmarking results demonstrate that Nydus images significantly o
4848
## Ecosystem
4949
### Nydus tools
5050

51-
| Tool | Description |
52-
| ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
51+
| Tool | Description |
52+
| -------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
5353
| [nydusd](https://github.com/dragonflyoss/nydus/blob/master/docs/nydusd.md) | Nydus user-space daemon, it processes all fscache/FUSE messages from the kernel and parses Nydus images to fullfil those requests |
5454
| [nydus-image](https://github.com/dragonflyoss/nydus/blob/master/docs/nydus-image.md) | Convert a single layer of OCI format container image into a nydus format container image generating meta part file and data part file respectively |
5555
| [nydusify](https://github.com/dragonflyoss/nydus/blob/master/docs/nydusify.md) | It pulls OCI image down and unpack it, invokes `nydus-image create` to convert image and then pushes the converted image back to registry and data storage |
5656
| [nydusctl](https://github.com/dragonflyoss/nydus/blob/master/docs/nydus-image.md) | Nydusd CLI client (`nydus-image inspect`), query daemon's working status/metrics and configure it |
57-
| [nydus-docker-graphdriver](https://github.com/nydusaccelerator/docker-nydus-graphdriver) | [Experimental] Works as a `docker` remote graph driver to control how images and containers are stored and managed |
57+
| [nydus-docker-graphdriver](https://github.com/nydusaccelerator/docker-nydus-graphdriver) | [Experimental] Works as a `docker` remote graph driver to control how images and containers are stored and managed |
5858
| [nydus-overlayfs](https://github.com/dragonflyoss/nydus/tree/master/contrib/nydus-overlayfs) | `Containerd` mount helper to invoke overlayfs mount with tweaking mount options a bit. So nydus prerequisites can be passed to vm-based runtime |
59-
| [nydus-backend-proxy](./contrib/nydus-backend-proxy/README.md) | A simple HTTP server to serve local directory as a blob backend for nydusd |
59+
| [nydus-backend-proxy](./contrib/nydus-backend-proxy/README.md) | A simple HTTP server to serve local directory as a blob backend for nydusd |
6060

6161
### Supported platforms
6262

6363
| Type | Platform | Description | Status |
6464
| ------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------ |
6565
| Storage | Registry/OSS/S3/NAS | Support for OCI-compatible distribution implementations such as Docker Hub, Harbor, Github GHCR, Aliyun ACR, NAS, and Aliyun OSS-like object storage service | ✅ |
6666
| Storage/Build | [Harbor](https://github.com/goharbor/acceleration-service) | Provides a general service for Harbor to support acceleration image conversion based on kinds of accelerator like Nydus and eStargz etc | ✅ |
67-
| Distribution | [Dragonfly](https://github.com/dragonflyoss/dragonfly) | Improve the runtime performance of Nydus image even further with the Dragonfly P2P data distribution system | ✅ |
68-
| Build | [Buildkit](https://github.com/nydusaccelerator/buildkit/blob/master/docs/nydus.md) | Provides the ability to build and export Nydus images directly from Dockerfile | ✅ |
67+
| Distribution | [Dragonfly](https://github.com/dragonflyoss/dragonfly) | Improve the runtime performance of Nydus image even further with the Dragonfly P2P data distribution system | ✅ |
68+
| Build | [Buildkit](https://github.com/nydusaccelerator/buildkit/blob/master/docs/nydus.md) | Provides the ability to build and export Nydus images directly from Dockerfile | ✅ |
6969
| Build/Runtime | [Nerdctl](https://github.com/containerd/nerdctl/blob/master/docs/nydus.md) | The containerd client to build or run (requires nydus snapshotter) Nydus image | ✅ |
70-
| Runtime | [Docker / Moby](https://github.com/dragonflyoss/nydus/blob/master/docs/docker-env-setup.md) | Run Nydus image in Docker container with containerd and nydus-snapshotter | ✅ |
70+
| Runtime | [Docker / Moby](https://github.com/dragonflyoss/nydus/blob/master/docs/docker-env-setup.md) | Run Nydus image in Docker container with containerd and nydus-snapshotter | ✅ |
7171
| Runtime | [Kubernetes](https://github.com/containerd/nydus-snapshotter/blob/main/docs/run_nydus_in_kubernetes.md) | Run Nydus image using CRI interface | ✅ |
7272
| Runtime | [Containerd](https://github.com/containerd/nydus-snapshotter) | Nydus Snapshotter, a containerd remote plugin to run Nydus image | ✅ |
7373
| Runtime | [CRI-O / Podman](https://github.com/containers/nydus-storage-plugin) | Run Nydus image with CRI-O or Podman | 🚧 |
@@ -169,6 +169,4 @@ Feel free to reach us via Slack or Dingtalk.
169169

170170
- **Twitter:** [@dragonfly_oss](https://twitter.com/dragonfly_oss)
171171

172-
- **Dingtalk:** [34971767](https://qr.dingtalk.com/action/joingroup?code=v1,k1,ioWGzuDZEIO10Bf+/ohz4RcQqAkW0MtOwoG1nbbMxQg=&_dt_no_comment=1&origin=11)
173-
174-
<img src="./misc/dingtalk.jpg" width="250" height="300"/>
172+
- **Dingtalk Group Number:** 34971767

‎api/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ serde = { version = "1.0.110", features = ["rc", "serde_derive"] }
2424
url = { version = "2.1.1", optional = true }
2525

2626
[dev-dependencies]
27-
vmm-sys-util = { version = "0.12.1" }
27+
vmm-sys-util = { workspace = true }
2828

2929
[features]
3030
error-backtrace = ["backtrace"]

‎builder/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
2020
serde_json = "1.0.53"
2121
sha2 = "0.10.2"
2222
tar = "0.4.40"
23-
vmm-sys-util = "0.12.1"
23+
vmm-sys-util = { workspace = true }
2424
xattr = "1.0.1"
2525
parse-size = "1.1.0"
2626

‎builder/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ pub use self::core::prefetch::{Prefetch, PrefetchPolicy};
4141
pub use self::core::tree::{MetadataTreeBuilder, Tree, TreeNode};
4242
pub use self::directory::DirectoryBuilder;
4343
pub use self::merge::Merger;
44+
pub use self::optimize_prefetch::generate_prefetch_file_info;
4445
pub use self::optimize_prefetch::update_ctx_from_bootstrap;
4546
pub use self::optimize_prefetch::OptimizePrefetch;
4647
pub use self::stargz::StargzBuilder;

0 commit comments

Comments
 (0)