From 7da0c4af1134f37899cde9851850a86da6820448 Mon Sep 17 00:00:00 2001 From: andygauge Date: Sat, 30 Nov 2024 14:13:48 -0800 Subject: [PATCH] issue-683 rewrite downloads clients --- src/web/clients/download/basic.md | 31 +++++--- src/web/clients/download/partial.md | 101 ++++++-------------------- src/web/clients/download/post-file.md | 14 +--- 3 files changed, 48 insertions(+), 98 deletions(-) diff --git a/src/web/clients/download/basic.md b/src/web/clients/download/basic.md index 3efae143..cf033d9a 100755 --- a/src/web/clients/download/basic.md +++ b/src/web/clients/download/basic.md @@ -7,21 +7,28 @@ a file over HTTP using [`reqwest::get`] asynchronously. Creates a target [`File`] with name obtained from [`Response::url`] within [`tempdir()`] and writes downloaded data into it with [`Writer::write_all`]. -The temporary directory is automatically removed on program exit. +The temporary directory is automatically removed on program exit as seen +in [`tempfile#examples`]. -```rust,edition2018,no_run -use error_chain::error_chain; +Add dependencies with cargo + +``` +cargo add anyhow reqwest tempfile tempfile tokio +``` + +Enable features in Cargo.toml + +``` +tokio = { version = "..", features = ["full"] } +``` + +```rust,edition2024,no_run + +use anyhow::Result; use std::io::Write; use std::fs::File; use tempfile::Builder; -error_chain! { - foreign_links { - Io(std::io::Error); - HttpRequest(reqwest::Error); - } -} - #[tokio::main] async fn main() -> Result<()> { let tmp_dir = Builder::new().prefix("example").tempdir()?; @@ -38,11 +45,12 @@ async fn main() -> Result<()> { println!("file to download: '{}'", fname); let fname = tmp_dir.path().join(fname); - println!("will be located under: '{:?}'", fname); + println!("will be located under: {}", fname.display()); File::create(fname)? }; let content = response.bytes().await?; dest.write_all(&content)?; + Ok(()) } ``` @@ -52,4 +60,5 @@ async fn main() -> Result<()> { [`Response::url`]: https://docs.rs/reqwest/*/reqwest/struct.Response.html#method.url [`tempfile::Builder`]: https://docs.rs/tempfile/*/tempfile/struct.Builder.html [`tempdir()`]: https://docs.rs/tempfile/*/tempfile/struct.Builder.html#method.tempdir +[`tempfile#examples`]: https://docs.rs/tempfile/latest/tempfile/#examples [`Writer::write_all`]: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all diff --git a/src/web/clients/download/partial.md b/src/web/clients/download/partial.md index 9b00bc06..11fc8782 100644 --- a/src/web/clients/download/partial.md +++ b/src/web/clients/download/partial.md @@ -1,99 +1,46 @@ -## Make a partial download with HTTP range headers +## Make a partial download with [![reqwest-badge]][reqwest] [![cat-net-badge]][cat-net] -Uses [`reqwest::blocking::Client::head`] to get the [Content-Length] of the response. +Uses [`reqwest::Client::head`] to get the [Content-Length] of the response. -The code then uses [`reqwest::blocking::Client::get`] to download the content in -chunks of 10240 bytes, while printing progress messages. This example uses the synchronous -reqwest module. The [Range] header specifies the chunk size and position. +The code then uses [`chunk`] to download the content in chunks writing +to a local file. -The Range header is defined in [RFC7233][HTTP Range RFC7233]. - -```rust,edition2018,no_run -use error_chain::error_chain; -use reqwest::header::{HeaderValue, CONTENT_LENGTH, RANGE}; -use reqwest::StatusCode; +```rust,edition2024,no_run +use anyhow::{Error, Result}; +use reqwest::header::{CONTENT_LENGTH}; use std::fs::File; +use std::io::Write; use std::str::FromStr; -error_chain! { - foreign_links { - Io(std::io::Error); - Reqwest(reqwest::Error); - Header(reqwest::header::ToStrError); - } -} - -struct PartialRangeIter { - start: u64, - end: u64, - buffer_size: u32, -} - -impl PartialRangeIter { - pub fn new(start: u64, end: u64, buffer_size: u32) -> Result { - if buffer_size == 0 { - Err("invalid buffer_size, give a value greater than zero.")?; - } - Ok(PartialRangeIter { - start, - end, - buffer_size, - }) - } -} - -impl Iterator for PartialRangeIter { - type Item = HeaderValue; - fn next(&mut self) -> Option { - if self.start > self.end { - None - } else { - let prev_start = self.start; - self.start += std::cmp::min(self.buffer_size as u64, self.end - self.start + 1); - Some(HeaderValue::from_str(&format!("bytes={}-{}", prev_start, self.start - 1)).expect("string provided by format!")) - } - } -} - -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let url = "https://httpbin.org/range/102400?duration=2"; - const CHUNK_SIZE: u32 = 10240; - let client = reqwest::blocking::Client::new(); - let response = client.head(url).send()?; - let length = response + let client = reqwest::Client::new(); + let header = client.head(url).send().await?; + let length = header .headers() - .get(CONTENT_LENGTH) - .ok_or("response doesn't include the content length")?; - let length = u64::from_str(length.to_str()?).map_err(|_| "invalid Content-Length header")?; + .get(CONTENT_LENGTH); + let length = u64::from_str( + length.expect("Content Length not provided").to_str()? + ).map_err(Error::msg)?; let mut output_file = File::create("download.bin")?; + let mut response = client.get(url).send().await?; println!("starting download..."); - for range in PartialRangeIter::new(0, length - 1, CHUNK_SIZE)? { - println!("range {:?}", range); - let mut response = client.get(url).header(RANGE, range).send()?; - - let status = response.status(); - if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT) { - error_chain::bail!("Unexpected server response: {}", status) - } - std::io::copy(&mut response, &mut output_file)?; + while let Some(chunk) = response.chunk().await? { + println!("Received chunk, writing to file"); + output_file.write_all(&chunk)?; } - - let content = response.text()?; - std::io::copy(&mut content.as_bytes(), &mut output_file)?; - println!("Finished with success!"); + println!("Finished with success! {} bytes", length); Ok(()) } ``` -[`reqwest::blocking::Client::get`]: https://docs.rs/reqwest/*/reqwest/blocking/struct.Client.html#method.get -[`reqwest::blocking::Client::head`]: https://docs.rs/reqwest/*/reqwest/blocking/struct.Client.html#method.head +[`reqwest::Client::head`]: https://docs.rs/reqwest/*/reqwest/blocking/struct.Client.html#method.head +[`chunk`]: https://docs.rs/reqwest/latest/reqwest/struct.Response.html#method.chunk [Content-Length]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Length -[Range]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range - -[HTTP Range RFC7233]: https://tools.ietf.org/html/rfc7233#section-3.1 diff --git a/src/web/clients/download/post-file.md b/src/web/clients/download/post-file.md index 79b9363f..9174fb26 100644 --- a/src/web/clients/download/post-file.md +++ b/src/web/clients/download/post-file.md @@ -9,21 +9,15 @@ content to send by reading the file, and [`RequestBuilder::send`] blocks until the file uploads and the response returns. [`read_to_string`] returns the response and displays in the console. -```rust,edition2018,no_run -use error_chain::error_chain; -use std::fs::File; +```rust,edition2024,no_run +use anyhow::Result; +use std::fs::{File, write}; use std::io::Read; - error_chain! { - foreign_links { - HttpRequest(reqwest::Error); - IoError(::std::io::Error); - } - } #[tokio::main] - async fn main() -> Result<()> { let paste_api = "https://paste.rs"; + write("message", "CONTENTS")?; let mut file = File::open("message")?; let mut contents = String::new();