Skip to content

Commit 960e4e1

Browse files
Feature gate object_store and reqwest (#71)
* pulled `coalesce_ranges` conditionally in-tree, feature-gated object_store and reqwest * undid coalesce_ranges addition * Update src/error.rs Co-authored-by: Kyle Barron <[email protected]> * Update Cargo.toml Co-authored-by: Kyle Barron <[email protected]> * added CI metadata to toml * fixed errors/warnings for all features: Added std::fs::File AsyncFileReader impl * added CI * removed AsyncFileReader impl for std::fs::File, added impls for tokio::fs::File * added required tokio features * revert util test changes * Add tokio wrapper * Only run `check` for all feature flag combinations, not `test` * remove wasm32 target * Minimal tokio features * add line to doc --------- Co-authored-by: Kyle Barron <[email protected]> Co-authored-by: Kyle Barron <[email protected]>
1 parent 6a7a2dc commit 960e4e1

File tree

5 files changed

+88
-35
lines changed

5 files changed

+88
-35
lines changed

.github/workflows/test.yml

+4-2
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@ jobs:
2626
- name: "clippy --all"
2727
run: cargo clippy --all --all-features --tests -- -D warnings
2828

29-
- name: "cargo check"
30-
run: cargo check --all --all-features
29+
- run: cargo install cargo-all-features
30+
31+
- name: Check all combinations of features can build
32+
run: cargo check-all-features
3133

3234
- name: "cargo test"
3335
run: |

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
.vscode
12
*.tif
23
*.buf
34

Cargo.toml

+21-6
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,29 @@ flate2 = "1.0.20"
1515
futures = "0.3.31"
1616
jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false }
1717
num_enum = "0.7.3"
18-
object_store = "0.12"
19-
# In the future we could make this feature-flagged, but for now we depend on
20-
# object_store which uses reqwest.
21-
reqwest = { version = "0.12", default-features = false }
18+
object_store = { version = "0.12", optional = true }
19+
reqwest = { version = "0.12", default-features = false, optional = true }
2220
thiserror = "1"
23-
tokio = { version = "1.43.0", optional = true }
21+
tokio = { version = "1.43.0", optional = true, default-features = false, features = [
22+
"io-util",
23+
"sync",
24+
] }
2425
weezl = "0.1.0"
2526

2627
[dev-dependencies]
28+
object_store = "0.12"
2729
tiff = "0.9.1"
28-
tokio = { version = "1.9", features = ["macros", "fs", "rt-multi-thread"] }
30+
tokio = { version = "1.9", features = [
31+
"macros",
32+
"fs",
33+
"rt-multi-thread",
34+
"io-util",
35+
] }
36+
37+
[features]
38+
default = ["object_store", "reqwest"]
39+
tokio = ["dep:tokio"]
40+
reqwest = ["dep:reqwest"]
41+
object_store = ["dep:object_store"]
42+
43+
[package.metadata.cargo-all-features]

src/error.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use thiserror::Error;
99
pub enum AsyncTiffError {
1010
/// End of file error.
1111
#[error("End of File: expected to read {0} bytes, got {1}")]
12-
EndOfFile(usize, usize),
12+
EndOfFile(u64, u64),
1313

1414
/// General error.
1515
#[error("General error: {0}")]
@@ -24,6 +24,7 @@ pub enum AsyncTiffError {
2424
JPEGDecodingError(#[from] jpeg::Error),
2525

2626
/// Error while fetching data using object store.
27+
#[cfg(feature = "object_store")]
2728
#[error(transparent)]
2829
ObjectStore(#[from] object_store::Error),
2930

@@ -32,6 +33,7 @@ pub enum AsyncTiffError {
3233
InternalTIFFError(#[from] crate::tiff::TiffError),
3334

3435
/// Reqwest error
36+
#[cfg(feature = "reqwest")]
3537
#[error(transparent)]
3638
ReqwestError(#[from] reqwest::Error),
3739

src/reader.rs

+59-26
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ use std::sync::Arc;
88
use byteorder::{BigEndian, LittleEndian, ReadBytesExt};
99
use bytes::buf::Reader;
1010
use bytes::{Buf, Bytes};
11-
use futures::future::{BoxFuture, FutureExt, TryFutureExt};
12-
use object_store::ObjectStore;
11+
use futures::future::{BoxFuture, FutureExt};
12+
use futures::TryFutureExt;
1313

1414
use crate::error::{AsyncTiffError, AsyncTiffResult};
1515

@@ -67,45 +67,75 @@ impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
6767
}
6868
}
6969

70-
// #[cfg(feature = "tokio")]
71-
// impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Debug + Send + Sync> AsyncFileReader
72-
// for T
73-
// {
74-
// fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
75-
// use tokio::io::{AsyncReadExt, AsyncSeekExt};
76-
77-
// async move {
78-
// self.seek(std::io::SeekFrom::Start(range.start)).await?;
79-
80-
// let to_read = (range.end - range.start).try_into().unwrap();
81-
// let mut buffer = Vec::with_capacity(to_read);
82-
// let read = self.take(to_read as u64).read_to_end(&mut buffer).await?;
83-
// if read != to_read {
84-
// return Err(AsyncTiffError::EndOfFile(to_read, read));
85-
// }
86-
87-
// Ok(buffer.into())
88-
// }
89-
// .boxed()
90-
// }
91-
// }
70+
/// A wrapper for things that implement [AsyncRead] and [AsyncSeek] to also implement
71+
/// [AsyncFileReader].
72+
///
73+
/// This wrapper is needed because `AsyncRead` and `AsyncSeek` require mutable access to seek and
74+
/// read data, while the `AsyncFileReader` trait requires immutable access to read data.
75+
///
76+
/// This wrapper stores the inner reader in a `Mutex`.
77+
///
78+
/// [AsyncRead]: tokio::io::AsyncRead
79+
/// [AsyncSeek]: tokio::io::AsyncSeek
80+
#[cfg(feature = "tokio")]
81+
#[derive(Debug)]
82+
pub struct TokioReader<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug>(
83+
tokio::sync::Mutex<T>,
84+
);
85+
86+
#[cfg(feature = "tokio")]
87+
impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug> TokioReader<T> {
88+
/// Create a new TokioReader from a reader.
89+
pub fn new(inner: T) -> Self {
90+
Self(tokio::sync::Mutex::new(inner))
91+
}
92+
}
93+
94+
#[cfg(feature = "tokio")]
95+
impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug> AsyncFileReader
96+
for TokioReader<T>
97+
{
98+
fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
99+
use std::io::SeekFrom;
100+
use tokio::io::{AsyncReadExt, AsyncSeekExt};
101+
102+
async move {
103+
let mut file = self.0.lock().await;
104+
105+
file.seek(SeekFrom::Start(range.start)).await?;
106+
107+
let to_read = range.end - range.start;
108+
let mut buffer = Vec::with_capacity(to_read as usize);
109+
let read = file.read(&mut buffer).await? as u64;
110+
if read != to_read {
111+
return Err(AsyncTiffError::EndOfFile(to_read, read));
112+
}
113+
114+
Ok(buffer.into())
115+
}
116+
.boxed()
117+
}
118+
}
92119

93120
/// An AsyncFileReader that reads from an [`ObjectStore`] instance.
121+
#[cfg(feature = "object_store")]
94122
#[derive(Clone, Debug)]
95123
pub struct ObjectReader {
96-
store: Arc<dyn ObjectStore>,
124+
store: Arc<dyn object_store::ObjectStore>,
97125
path: object_store::path::Path,
98126
}
99127

128+
#[cfg(feature = "object_store")]
100129
impl ObjectReader {
101130
/// Creates a new [`ObjectReader`] for the provided [`ObjectStore`] and path
102131
///
103132
/// [`ObjectMeta`] can be obtained using [`ObjectStore::list`] or [`ObjectStore::head`]
104-
pub fn new(store: Arc<dyn ObjectStore>, path: object_store::path::Path) -> Self {
133+
pub fn new(store: Arc<dyn object_store::ObjectStore>, path: object_store::path::Path) -> Self {
105134
Self { store, path }
106135
}
107136
}
108137

138+
#[cfg(feature = "object_store")]
109139
impl AsyncFileReader for ObjectReader {
110140
fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
111141
let range = range.start as _..range.end as _;
@@ -134,19 +164,22 @@ impl AsyncFileReader for ObjectReader {
134164
}
135165

136166
/// An AsyncFileReader that reads from a URL using reqwest.
167+
#[cfg(feature = "reqwest")]
137168
#[derive(Debug, Clone)]
138169
pub struct ReqwestReader {
139170
client: reqwest::Client,
140171
url: reqwest::Url,
141172
}
142173

174+
#[cfg(feature = "reqwest")]
143175
impl ReqwestReader {
144176
/// Construct a new ReqwestReader from a reqwest client and URL.
145177
pub fn new(client: reqwest::Client, url: reqwest::Url) -> Self {
146178
Self { client, url }
147179
}
148180
}
149181

182+
#[cfg(feature = "reqwest")]
150183
impl AsyncFileReader for ReqwestReader {
151184
fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
152185
let url = self.url.clone();

0 commit comments

Comments
 (0)