zarr-datafusion/Cargo.toml at main · stratoscale-io/zarr-datafusion · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
[package]
name = "zarr-datafusion"
version = "0.1.1"
edition = "2021"
license = "Apache-2.0"
rust-version = "1.91"
description = "Extending DataFusion to do SQL queries on Zarr data."
repository = "https://github.com/stratoscale-io/zarr-datafusion"
homepage = "https://stratoscale.io"
documentation = "https://docs.rs/zarr-datafusion"
readme = "README.md"
keywords = ["zarr", "datafusion", "sql", "arrow", "scientific-data"]
categories = ["database", "science", "data-structures"]

[dependencies]
arrow = "58"
async-trait = "0.1"
bytes = "1.11"
chrono = "0.4"
datafusion = "54.0.0"
futures = "0.3.31"
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
zarrs = { version = "0.22.10", features = ["ndarray", "async", "zlib"] }
zarrs_object_store = { version = "0.6", features = ["gcp", "aws", "http"] }
# Icechunk store support. Optional: only pulled in by the `icechunk` feature
# (~56 transitive crates), so the default build stays lean. zarrs_icechunk
# provides AsyncIcechunkStore, a zarrs storage backend (zarrs_storage 0.4 traits
# — the same our zarrs 0.22 uses, so no zarrs bump is needed). icechunk itself is
# a direct dep because we call its Repository / storage constructors. Lean
# backend set: local fs (synthetic spike), GCS (the repo lives there), and S3
# (virtual HDF5 chunks on noaa-oar-mlwp-data) — avoiding the heavy aws-sdk `s3`
# and azure/http backends.
icechunk = { version = "2.0.1", default-features = false, features = ["object-store-fs", "object-store-gcs", "object-store-s3"], optional = true }
zarrs_icechunk = { version = "0.5", default-features = false, optional = true }
parquet = { version = "58", default-features = false, features = ["arrow", "async"] }
glob = "0.3"
url = "2"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
datafusion-proto = "54.0.0"
prost = "0.14.4"
serde = { version = "1.0.228", features = ["derive"] }

# Distributed execution (datafusion-distributed). Optional: only pulled in by the
# `distributed` feature, which also gates the `worker` and `head` binaries.
datafusion-distributed = { version = "2", optional = true }
tonic = { version = "0.14", optional = true }

[features]
# Enables the datafusion-distributed worker/head binaries.
distributed = ["dep:datafusion-distributed", "dep:tonic"]
# Enables icechunk store support (reading icechunk repos, incl. virtual chunks
# into archival NetCDF/HDF5). Pulls the icechunk + zarrs_icechunk crate trees.
icechunk = ["dep:icechunk", "dep:zarrs_icechunk"]

[[example]]
name = "icechunk_m0_probe"
path = "examples/icechunk_m0_probe.rs"
required-features = ["icechunk"]

[[example]]
name = "icechunk_read"
path = "examples/icechunk_read.rs"
required-features = ["icechunk"]

[[example]]
name = "icechunk_read_virtual"
path = "examples/icechunk_read_virtual.rs"
required-features = ["icechunk"]

[[example]]
name = "icechunk_read_cira"
path = "examples/icechunk_read_cira.rs"
required-features = ["icechunk"]

[[example]]
name = "query_era5"
path = "examples/query_era5.rs"

[[example]]
name = "query_synthetic"
path = "examples/query_synthetic.rs"

[[bin]]
name = "zarr-cli"
path = "src/bin/zarr_cli/main.rs"

[[bin]]
name = "worker"
path = "src/bin/worker.rs"
required-features = ["distributed"]

[[bin]]
name = "head"
path = "src/bin/head.rs"
required-features = ["distributed"]

[dependencies.rustyline]
version = "15"
features = ["derive"]

[dependencies.serde_json]
version = "1"

[dependencies.sqlparser]
version = "0.53"

[dependencies.nu-ansi-term]
version = "0.50"

[dev-dependencies]
proptest = "1.0"