diff --git a/examples/fetch-rs/Cargo.lock b/examples/fetch-rs/Cargo.lock index 6e87da3c..24bf2c19 100644 --- a/examples/fetch-rs/Cargo.lock +++ b/examples/fetch-rs/Cargo.lock @@ -3,16 +3,24 @@ version = 4 [[package]] -name = "ahash" -version = "0.8.11" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", - "zerocopy", + "alloc-no-stdlib", ] [[package]] @@ -53,6 +61,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.8.0" @@ -60,16 +74,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] -name = "bumpalo" -version = "3.17.0" +name = "brotli" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] [[package]] -name = "byteorder" -version = "1.5.0" +name = "brotli-decompressor" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytes" @@ -113,60 +142,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "cssparser" -version = "0.31.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be" -dependencies = [ - "cssparser-macros", - "dtoa-short", - "itoa", - "phf 0.11.3", - "smallvec", -] - -[[package]] -name = "cssparser-macros" -version = "0.6.1" +name = "crc32fast" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "quote", - "syn 2.0.98", + "cfg-if", ] [[package]] -name = "derive_more" -version = "0.99.19" +name = "displaydoc" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", "syn 2.0.98", ] -[[package]] -name = "dtoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" - -[[package]] -name = "dtoa-short" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" -dependencies = [ - "dtoa", -] - -[[package]] -name = "ego-tree" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642" - [[package]] name = "equivalent" version = "1.0.1" @@ -177,13 +171,29 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" name = "fetch-rs" version = "0.1.0" dependencies = [ - "scraper", + "base64", + "brotli", + "flate2", + "http 0.2.12", + "mime", + "serde", "serde_json", "spin-executor", "spin-sdk", + "url", "wit-bindgen-rt", ] +[[package]] +name = "flate2" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -192,23 +202,13 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] -[[package]] -name = "futf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" -dependencies = [ - "mac", - "new_debug_unreachable", -] - [[package]] name = "futures" version = "0.3.31" @@ -298,35 +298,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "hashbrown" version = "0.15.2" @@ -343,17 +314,14 @@ dependencies = [ ] [[package]] -name = "html5ever" -version = "0.26.0" +name = "http" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "log", - "mac", - "markup5ever", - "proc-macro2", - "quote", - "syn 1.0.109", + "bytes", + "fnv", + "itoa", ] [[package]] @@ -391,221 +359,210 @@ dependencies = [ ] [[package]] -name = "id-arena" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" - -[[package]] -name = "indexmap" -version = "2.7.1" +name = "icu_collections" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ - "equivalent", - "hashbrown", - "serde", + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "itoa" -version = "1.0.14" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ - "once_cell", - "wasm-bindgen", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "leb128" -version = "0.2.5" +name = "icu_normalizer" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] [[package]] -name = "libc" -version = "0.2.169" +name = "icu_normalizer_data" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] -name = "lock_api" -version = "0.4.12" +name = "icu_properties" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ - "autocfg", - "scopeguard", + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", ] [[package]] -name = "log" -version = "0.4.25" +name = "icu_properties_data" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] -name = "mac" -version = "0.1.1" +name = "icu_provider" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] [[package]] -name = "markup5ever" -version = "0.11.0" +name = "id-arena" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" -dependencies = [ - "log", - "phf 0.10.1", - "phf_codegen", - "string_cache", - "string_cache_codegen", - "tendril", -] +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" [[package]] -name = "memchr" -version = "2.7.4" +name = "idna" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] [[package]] -name = "new_debug_unreachable" -version = "1.0.6" +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] [[package]] -name = "num-traits" -version = "0.2.19" +name = "indexmap" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ - "autocfg", + "equivalent", + "hashbrown", + "serde", ] [[package]] -name = "once_cell" -version = "1.20.2" +name = "itoa" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] -name = "parking_lot" -version = "0.12.3" +name = "js-sys" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ - "lock_api", - "parking_lot_core", + "once_cell", + "wasm-bindgen", ] [[package]] -name = "parking_lot_core" -version = "0.9.10" +name = "leb128" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] -name = "percent-encoding" -version = "2.3.1" +name = "libc" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] -name = "phf" -version = "0.10.1" +name = "litemap" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" -dependencies = [ - "phf_shared 0.10.0", -] +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] -name = "phf" -version = "0.11.3" +name = "log" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_macros", - "phf_shared 0.11.3", -] +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] -name = "phf_codegen" -version = "0.10.0" +name = "memchr" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", -] +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] -name = "phf_generator" -version = "0.10.0" +name = "mime" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" -dependencies = [ - "phf_shared 0.10.0", - "rand", -] +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -name = "phf_generator" -version = "0.11.3" +name = "miniz_oxide" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ - "phf_shared 0.11.3", - "rand", + "adler2", ] [[package]] -name = "phf_macros" -version = "0.11.3" +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", - "proc-macro2", - "quote", - "syn 2.0.98", + "autocfg", ] [[package]] -name = "phf_shared" -version = "0.10.0" +name = "once_cell" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher 0.3.11", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] -name = "phf_shared" -version = "0.11.3" +name = "percent-encoding" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher 1.0.1", -] +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" @@ -620,20 +577,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] -name = "ppv-lite86" -version = "0.2.20" +name = "potential_utf" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ - "zerocopy", + "zerovec", ] -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - [[package]] name = "proc-macro2" version = "1.0.93" @@ -652,45 +603,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "redox_syscall" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" -dependencies = [ - "bitflags", -] - [[package]] name = "routefinder" version = "0.5.4" @@ -713,47 +625,6 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "scraper" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585480e3719b311b78a573db1c9d9c4c1f8010c2dee4cc59c2efe58ea4dbc3e1" -dependencies = [ - "ahash", - "cssparser", - "ego-tree", - "getopts", - "html5ever", - "once_cell", - "selectors", - "tendril", -] - -[[package]] -name = "selectors" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" -dependencies = [ - "bitflags", - "cssparser", - "derive_more", - "fxhash", - "log", - "new_debug_unreachable", - "phf 0.10.1", - "phf_codegen", - "precomputed-hash", - "servo_arc", - "smallvec", -] - [[package]] name = "semver" version = "1.0.25" @@ -792,33 +663,12 @@ dependencies = [ "serde", ] -[[package]] -name = "servo_arc" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - -[[package]] -name = "siphasher" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" - [[package]] name = "slab" version = "0.4.9" @@ -899,7 +749,7 @@ dependencies = [ "chrono", "form_urlencoded", "futures", - "http", + "http 1.2.0", "once_cell", "routefinder", "serde", @@ -922,31 +772,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "string_cache" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938d512196766101d333398efde81bc1f37b00cb42c2f8350e5df639f040bbbe" -dependencies = [ - "new_debug_unreachable", - "parking_lot", - "phf_shared 0.11.3", - "precomputed-hash", - "serde", -] - -[[package]] -name = "string_cache_codegen" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244292f3441c89febe5b5bdfbb6863aeaf4f64da810ea3050fd927b27b8d92ce" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", - "proc-macro2", - "quote", -] - [[package]] name = "syn" version = "1.0.109" @@ -970,14 +795,14 @@ dependencies = [ ] [[package]] -name = "tendril" -version = "0.4.3" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ - "futf", - "mac", - "utf-8", + "proc-macro2", + "quote", + "syn 2.0.98", ] [[package]] @@ -1000,6 +825,16 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "unicode-ident" version = "1.0.16" @@ -1012,12 +847,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-xid" version = "0.2.6" @@ -1025,22 +854,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] -name = "utf-8" -version = "0.7.6" +name = "url" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] [[package]] -name = "version_check" -version = "0.9.5" +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "version_check" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wasm-bindgen" @@ -1323,20 +1158,83 @@ dependencies = [ ] [[package]] -name = "zerocopy" -version = "0.7.35" +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ - "byteorder", - "zerocopy-derive", + "yoke", + "zerofrom", + "zerovec-derive", ] [[package]] -name = "zerocopy-derive" -version = "0.7.35" +name = "zerovec-derive" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", diff --git a/examples/fetch-rs/Cargo.toml b/examples/fetch-rs/Cargo.toml index b0644ab8..78220f50 100644 --- a/examples/fetch-rs/Cargo.toml +++ b/examples/fetch-rs/Cargo.toml @@ -5,11 +5,17 @@ edition = "2021" license = "MIT" [dependencies] +base64 = "0.22.1" +brotli = "3.4.0" +flate2 = { version = "1.0.34", default-features = false, features = ["rust_backend"] } +mime = "0.3.17" +http = "0.2.12" +serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0.137" spin-executor = "3.0.1" spin-sdk = "3.0.1" +url = "2.5.4" wit-bindgen-rt = { version = "0.26.0", features = ["bitflags"] } -scraper = "0.18.1" [lib] crate-type = ["cdylib"] diff --git a/examples/fetch-rs/src/lib.rs b/examples/fetch-rs/src/lib.rs index 4fddde91..52c5e1e4 100644 --- a/examples/fetch-rs/src/lib.rs +++ b/examples/fetch-rs/src/lib.rs @@ -1,99 +1,790 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -use spin_sdk::http::{send, Request, Response}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use brotli::Decompressor; +use flate2::read::{GzDecoder, ZlibDecoder}; +use http::StatusCode as HttpStatusCode; +use mime::Mime; +use serde::Serialize; +use serde_json::Value; +use spin_sdk::http::{Method, Request, Response}; +use std::borrow::Cow; +use std::collections::HashSet; +use std::io::Read; +use std::time::Instant; +use url::Url; -#[allow(warnings)] mod bindings; use bindings::Guest; -use serde_json::Value; struct Component; +const DEFAULT_MAX_REDIRECTS: usize = 5; +const DEFAULT_MAX_TEXT_BYTES: usize = 128 * 1024; +const DEFAULT_MAX_BINARY_BYTES: usize = 256 * 1024; +const DEFAULT_USER_AGENT: &str = "mcp-fetch/1.0 (+https://github.com/microsoft/wassette)"; +const DEFAULT_BROTLI_BUFFER_SIZE: usize = 4096; +const ENV_MAX_REDIRECTS: &str = "FETCH_MAX_REDIRECTS"; +const ENV_MAX_TEXT_BYTES: &str = "FETCH_MAX_TEXT_BYTES"; +const ENV_MAX_BINARY_BYTES: &str = "FETCH_MAX_BINARY_BYTES"; +const ENV_TIMEOUT_MS: &str = "FETCH_TIMEOUT_MS"; +const ENV_USER_AGENT: &str = "FETCH_USER_AGENT"; + +#[derive(Clone)] +struct FetchOptions { + max_redirects: usize, + max_text_bytes: usize, + max_binary_bytes: usize, + timeout_ms: Option, + user_agent: Option, +} + +impl Default for FetchOptions { + fn default() -> Self { + Self { + max_redirects: DEFAULT_MAX_REDIRECTS, + max_text_bytes: DEFAULT_MAX_TEXT_BYTES, + max_binary_bytes: DEFAULT_MAX_BINARY_BYTES, + timeout_ms: None, + user_agent: None, + } + } +} + +impl FetchOptions { + fn from_env() -> Self { + let mut options = Self::default(); + + if let Ok(value) = std::env::var(ENV_MAX_REDIRECTS) { + if let Ok(parsed) = value.parse::() { + options.max_redirects = parsed; + } + } + + if let Ok(value) = std::env::var(ENV_MAX_TEXT_BYTES) { + if let Ok(parsed) = value.parse::() { + options.max_text_bytes = parsed; + } + } + + if let Ok(value) = std::env::var(ENV_MAX_BINARY_BYTES) { + if let Ok(parsed) = value.parse::() { + options.max_binary_bytes = parsed; + } + } + + if let Ok(value) = std::env::var(ENV_TIMEOUT_MS) { + if let Ok(parsed) = value.parse::() { + options.timeout_ms = Some(parsed); + } + } + + if let Ok(value) = std::env::var(ENV_USER_AGENT) { + if !value.trim().is_empty() { + options.user_agent = Some(value); + } + } + + options + } + + fn max_redirects(&self) -> usize { + self.max_redirects + } + + fn user_agent(&self) -> &str { + self.user_agent + .as_deref() + .unwrap_or(DEFAULT_USER_AGENT) + } + + fn timeout_ms(&self) -> Option { + self.timeout_ms + } + + fn max_text_bytes(&self) -> usize { + self.max_text_bytes + } + + fn max_binary_bytes(&self) -> usize { + self.max_binary_bytes + } + + fn brotli_buffer(&self) -> usize { + DEFAULT_BROTLI_BUFFER_SIZE + } +} + impl Guest for Component { fn fetch(url: String) -> Result { spin_executor::run(async move { - let request = Request::get(url); - let response: Response = send(request).await.map_err(|e| e.to_string())?; - let status = response.status(); - if !(200..300).contains(status) { - return Err(format!("Request failed with status code: {}", status)); + let options = FetchOptions::from_env(); + match fetch_impl(url, options).await { + Ok(success) => serde_json::to_string_pretty(&success).map_err(|e| e.to_string()), + Err(error) => match serde_json::to_string_pretty(&error) { + Ok(json) => Err(json), + Err(serde_err) => Err( + serde_json::json!({ + "error": format!("Failed to serialize fetch error: {}", serde_err) + }) + .to_string(), + ), + }, } - let body = String::from_utf8_lossy(response.body()); - - if let Some(content_type) = response.header("content-type").and_then(|v| v.as_str()) { - if content_type.contains("application/json") { - let json: Value = serde_json::from_str(&body).map_err(|e| e.to_string())?; - return Ok(json_to_markdown(&json)); - } else if content_type.contains("text/html") { - return Ok(html_to_markdown(&body)); + }) + } +} + +#[derive(Serialize)] +struct FetchSuccess { + final_url: String, + status: u16, + status_text: Option, + headers: Vec, + content_type: Option, + content_encoding: Option, + redirect_chain: Vec, + body: Body, + warnings: Vec, + metrics: Metrics, +} + +#[derive(Serialize)] +struct FetchError { + error: String, + url: String, + status: Option, + status_text: Option, + headers: Vec, + content_type: Option, + content_encoding: Option, + redirect_chain: Vec, + body: Option, + warnings: Vec, + metrics: Option, +} + +#[derive(Serialize, Clone)] +struct HeaderEntry { + name: String, + value: String, +} + +#[derive(Serialize, Clone)] +struct RedirectHop { + url: String, + status: u16, + location: String, +} + +#[derive(Serialize, Clone, Copy)] +struct Metrics { + elapsed_ms: u128, + decoded_body_bytes: usize, +} + +#[derive(Serialize)] +#[serde(tag = "format", rename_all = "snake_case")] +enum Body { + Empty, + Json { + size: usize, + truncated: bool, + value: Value, + }, + Text { + size: usize, + truncated: bool, + encoding: String, + content: String, + }, + Binary { + size: usize, + truncated: bool, + encoding: String, + base64: String, + }, +} + +impl Metrics { + fn from_elapsed(duration: std::time::Duration, decoded_body_bytes: usize) -> Self { + Self { + elapsed_ms: duration.as_millis(), + decoded_body_bytes, + } + } +} + +impl FetchError { + fn invalid_url(url: String, cause: String) -> Self { + Self { + error: format!("Invalid URL: {}", cause), + url, + status: None, + status_text: None, + headers: Vec::new(), + content_type: None, + content_encoding: None, + redirect_chain: Vec::new(), + body: None, + warnings: Vec::new(), + metrics: None, + } + } + + fn network(url: String, redirect_chain: Vec, cause: String, metrics: Metrics) -> Self { + Self { + error: format!("Network error: {}", cause), + url, + status: None, + status_text: None, + headers: Vec::new(), + content_type: None, + content_encoding: None, + redirect_chain, + body: None, + warnings: Vec::new(), + metrics: Some(metrics), + } + } + + fn redirect_limit( + url: String, + mut redirect_chain: Vec, + status: u16, + status_text: Option, + headers: Vec, + metrics: Metrics, + max_redirects: usize, + ) -> Self { + if let Some(last) = redirect_chain.last_mut() { + last.status = status; + } + + Self { + error: format!("Redirect limit of {} exceeded", max_redirects), + url, + status: Some(status), + status_text, + headers, + content_type: None, + content_encoding: None, + redirect_chain, + body: None, + warnings: vec!["Too many redirects".to_string()], + metrics: Some(metrics), + } + } + + fn redirect_loop( + url: String, + redirect_chain: Vec, + metrics: Metrics, + ) -> Self { + Self { + error: "Detected redirect loop".to_string(), + url, + status: None, + status_text: None, + headers: Vec::new(), + content_type: None, + content_encoding: None, + redirect_chain, + body: None, + warnings: vec!["Redirect loop detected".to_string()], + metrics: Some(metrics), + } + } + + fn timeout(url: String, redirect_chain: Vec, metrics: Metrics) -> Self { + Self { + error: "Request timed out".to_string(), + url, + status: None, + status_text: None, + headers: Vec::new(), + content_type: None, + content_encoding: None, + redirect_chain, + body: None, + warnings: Vec::new(), + metrics: Some(metrics), + } + } + + fn redirect_resolution( + url: String, + mut redirect_chain: Vec, + location: String, + cause: String, + metrics: Metrics, + ) -> Self { + if let Some(last) = redirect_chain.last_mut() { + last.location = location.clone(); + } + + Self { + error: format!("Failed to resolve redirect location '{}': {}", location, cause), + url, + status: None, + status_text: None, + headers: Vec::new(), + content_type: None, + content_encoding: None, + redirect_chain, + body: None, + warnings: Vec::new(), + metrics: Some(metrics), + } + } + + fn processing( + url: String, + redirect_chain: Vec, + status: Option, + status_text: Option, + headers: Vec, + content_type: Option, + content_encoding: Option, + cause: String, + warnings: Vec, + metrics: Metrics, + ) -> Self { + Self { + error: cause, + url, + status, + status_text, + headers, + content_type, + content_encoding, + redirect_chain, + body: None, + warnings, + metrics: Some(metrics), + } + } + + fn http( + url: String, + status: u16, + status_text: Option, + headers: Vec, + content_type: Option, + content_encoding: Option, + redirect_chain: Vec, + body: Body, + warnings: Vec, + metrics: Metrics, + ) -> Self { + Self { + error: format!("HTTP {} response", status), + url, + status: Some(status), + status_text, + headers, + content_type, + content_encoding, + redirect_chain, + body: Some(body), + warnings, + metrics: Some(metrics), + } + } +} + +async fn fetch_impl(initial_url: String, options: FetchOptions) -> Result { + let parsed_url = Url::parse(&initial_url) + .map_err(|e| FetchError::invalid_url(initial_url.clone(), e.to_string()))?; + + if !matches!(parsed_url.scheme(), "http" | "https") { + return Err(FetchError::invalid_url( + initial_url, + "Unsupported URL scheme (only http/https allowed)".to_string(), + )); + } + + let mut current_url = parsed_url; + let mut redirect_chain = Vec::new(); + let mut redirect_count = 0usize; + let mut visited = HashSet::new(); + visited.insert(current_url.to_string()); + + let start = Instant::now(); + + loop { + let request = build_request(current_url.as_str(), &options); + + let response: Response = match spin_sdk::http::send(request).await { + Ok(resp) => resp, + Err(err) => { + return Err(FetchError::network( + current_url.to_string(), + redirect_chain, + err.to_string(), + Metrics::from_elapsed(start.elapsed(), 0), + )) + } + }; + + let status_code = *response.status(); + let status_text = HttpStatusCode::from_u16(status_code) + .ok() + .and_then(|status| status.canonical_reason().map(|reason| reason.to_string())); + let headers = collect_headers(&response); + + if is_redirect_status(status_code) { + if let Some(location) = response.header("location").and_then(|h| h.as_str()) { + if redirect_count >= options.max_redirects() { + return Err(FetchError::redirect_limit( + current_url.to_string(), + redirect_chain, + status_code, + status_text, + headers, + Metrics::from_elapsed(start.elapsed(), 0), + options.max_redirects(), + )); + } + + let resolved = match resolve_redirect(¤t_url, location) { + Ok(url) => url, + Err(err) => { + return Err(FetchError::redirect_resolution( + current_url.to_string(), + redirect_chain, + location.to_string(), + err.to_string(), + Metrics::from_elapsed(start.elapsed(), 0), + )) + } + }; + + if visited.contains(resolved.as_str()) { + return Err(FetchError::redirect_loop( + current_url.to_string(), + redirect_chain, + Metrics::from_elapsed(start.elapsed(), 0), + )); } + + redirect_chain.push(RedirectHop { + url: current_url.to_string(), + status: status_code, + location: location.to_string(), + }); + + current_url = resolved; + redirect_count += 1; + visited.insert(current_url.to_string()); + continue; } + } - Ok(body.into_owned()) - }) + let content_type = response + .header("content-type") + .and_then(|h| h.as_str()) + .map(|s| s.to_string()); + let content_encoding = response + .header("content-encoding") + .and_then(|h| h.as_str()) + .map(|s| s.to_string()); + + let (decoded_body, mut decode_warnings) = + decode_body(response.body(), content_encoding.as_deref(), options.brotli_buffer()) + .map_err(|cause| { + FetchError::processing( + current_url.to_string(), + redirect_chain.clone(), + Some(status_code), + status_text.clone(), + headers.clone(), + content_type.clone(), + content_encoding.clone(), + format!("Failed to decode body: {}", cause), + Vec::new(), + Metrics::from_elapsed(start.elapsed(), 0), + ) + })?; + + let (body, mut body_warnings) = + build_body_representation(&decoded_body, content_type.as_deref(), &options); + + let mut warnings = Vec::new(); + warnings.append(&mut decode_warnings); + warnings.append(&mut body_warnings); + + let metrics = Metrics::from_elapsed(start.elapsed(), decoded_body.len()); + + if let Some(limit) = options.timeout_ms() { + if metrics.elapsed_ms > limit as u128 { + return Err(FetchError::timeout( + current_url.to_string(), + redirect_chain, + metrics, + )); + } + } + + if !(200..300).contains(&status_code) { + return Err(FetchError::http( + current_url.to_string(), + status_code, + status_text, + headers, + content_type, + content_encoding, + redirect_chain, + body, + warnings, + metrics, + )); + } + + return Ok(FetchSuccess { + final_url: current_url.to_string(), + status: status_code, + status_text, + headers, + content_type, + content_encoding, + redirect_chain, + body, + warnings, + metrics, + }); } } -fn html_to_markdown(html: &str) -> String { - let mut markdown = String::new(); - let fragment = scraper::Html::parse_fragment(html); - let text_selector = scraper::Selector::parse("h1, h2, h3, h4, h5, h6, p, a, div").unwrap(); - - for element in fragment.select(&text_selector) { - let tag_name = element.value().name(); - let text = element.text().collect::>().join(" ").trim().to_string(); - - if text.is_empty() { - continue; - } - - match tag_name { - "h1" => markdown.push_str(&format!("# {}\n\n", text)), - "h2" => markdown.push_str(&format!("## {}\n\n", text)), - "h3" => markdown.push_str(&format!("### {}\n\n", text)), - "h4" => markdown.push_str(&format!("#### {}\n\n", text)), - "h5" => markdown.push_str(&format!("##### {}\n\n", text)), - "h6" => markdown.push_str(&format!("###### {}\n\n", text)), - "p" => markdown.push_str(&format!("{}\n\n", text)), - "a" => { - if let Some(href) = element.value().attr("href") { - markdown.push_str(&format!("[{}]({})\n\n", text, href)); - } else { - markdown.push_str(&format!("{}\n\n", text)); +fn build_request(url: &str, options: &FetchOptions) -> Request { + let mut builder = Request::builder(); + builder + .method(Method::Get) + .uri(url) + .header("User-Agent", options.user_agent()) + .header("Accept", "*/*") + .header("Accept-Encoding", "gzip, deflate, br"); + builder.build() +} + +fn is_redirect_status(status: u16) -> bool { + matches!(status, 300..=399) +} + +fn resolve_redirect(base: &Url, location: &str) -> Result { + Url::parse(location).or_else(|_| base.join(location)) +} + +fn collect_headers(response: &Response) -> Vec { + response + .headers() + .map(|(name, value)| HeaderEntry { + name: name.to_string(), + value: value + .as_str() + .map(|s| s.to_string()) + .unwrap_or_else(|| String::from_utf8_lossy(value.as_bytes()).into_owned()), + }) + .collect() +} + +fn decode_body( + body: &[u8], + content_encoding: Option<&str>, + brotli_buffer: usize, +) -> Result<(Vec, Vec), String> { + let mut warnings = Vec::new(); + let mut data = body.to_vec(); + + if let Some(header) = content_encoding { + let encodings: Vec = header + .split(',') + .map(|part| part.trim().to_ascii_lowercase()) + .filter(|part| !part.is_empty()) + .collect(); + + for encoding in encodings.into_iter().rev() { + match encoding.as_str() { + "gzip" | "x-gzip" => { + let mut decoder = GzDecoder::new(data.as_slice()); + let mut decoded = Vec::new(); + decoder + .read_to_end(&mut decoded) + .map_err(|e| format!("Failed to decode gzip body: {}", e))?; + data = decoded; + } + "deflate" => { + let mut decoder = ZlibDecoder::new(data.as_slice()); + let mut decoded = Vec::new(); + decoder + .read_to_end(&mut decoded) + .map_err(|e| format!("Failed to decode deflate body: {}", e))?; + data = decoded; } - }, - _ => markdown.push_str(&format!("{}\n\n", text)), + "br" => { + let mut decoder = Decompressor::new(data.as_slice(), brotli_buffer); + let mut decoded = Vec::new(); + decoder + .read_to_end(&mut decoded) + .map_err(|e| format!("Failed to decode brotli body: {}", e))?; + data = decoded; + } + "identity" | "" => {} + other => warnings.push(format!( + "Unsupported content-encoding '{}'; returning raw body", + other + )), + } } } - markdown.trim().to_string() + Ok((data, warnings)) } -fn json_to_markdown(value: &Value) -> String { - match value { - Value::Object(map) => { - let mut markdown = String::new(); - for (key, val) in map { - markdown.push_str(&format!("### {}\n\n{}\n\n", key, json_to_markdown(val))); +fn build_body_representation( + body: &[u8], + content_type: Option<&str>, + options: &FetchOptions, +) -> (Body, Vec) { + if body.is_empty() { + return (Body::Empty, Vec::new()); + } + + let mut warnings = Vec::new(); + let size = body.len(); + let mime = content_type.and_then(|ct| ct.parse::().ok()); + + if let Some(mime) = mime.as_ref() { + if is_json_mime(mime) { + if let Ok(value) = serde_json::from_slice::(body) { + return (Body::Json { size, truncated: false, value }, warnings); } - markdown - } - Value::Array(arr) => { - let mut markdown = String::new(); - for (i, val) in arr.iter().enumerate() { - markdown.push_str(&format!("1. {}\n", json_to_markdown(val))); - if i < arr.len() - 1 { - markdown.push('\n'); + + if let Ok(text) = std::str::from_utf8(body) { + let mut values = Vec::new(); + for line in text.lines().map(str::trim).filter(|line| !line.is_empty()) { + match serde_json::from_str::(line) { + Ok(value) => values.push(value), + Err(_) => { + values.clear(); + break; + } + } + } + + if !values.is_empty() { + warnings.push("Interpreted body as newline-delimited JSON".to_string()); + return (Body::Json { size, truncated: false, value: Value::Array(values) }, warnings); } } - markdown + + warnings.push("Failed to parse response body as JSON; returning as text".to_string()); + } + + if is_text_mime(mime) { + return ( + build_text_body(body, size, options.max_text_bytes()), + warnings, + ); + } + } + + if looks_like_json(body) { + match serde_json::from_slice::(body) { + Ok(value) => return (Body::Json { size, truncated: false, value }, warnings), + Err(err) => warnings.push(format!("Failed to parse JSON content: {}", err)), + } + } + + if std::str::from_utf8(body).is_ok() { + return ( + build_text_body(body, size, options.max_text_bytes()), + warnings, + ); + } + + warnings.push("Response treated as binary data".to_string()); + ( + build_binary_body(body, size, options.max_binary_bytes()), + warnings, + ) +} + +fn build_text_body(body: &[u8], size: usize, limit: usize) -> Body { + let (clipped, truncated) = clip_bytes(body, limit); + let cow = String::from_utf8_lossy(&clipped); + let (content, encoding) = match cow { + Cow::Borrowed(_) => (cow.into_owned(), "utf-8".to_string()), + Cow::Owned(s) => (s, "lossy-utf-8".to_string()), + }; + + Body::Text { + size, + truncated, + encoding, + content, + } +} + +fn build_binary_body(body: &[u8], size: usize, limit: usize) -> Body { + let (clipped, truncated) = clip_bytes(body, limit); + let base64 = BASE64.encode(clipped); + Body::Binary { + size, + truncated, + encoding: "base64".to_string(), + base64, + } +} + +fn clip_bytes(data: &[u8], limit: usize) -> (Vec, bool) { + if data.len() > limit { + (data[..limit].to_vec(), true) + } else { + (data.to_vec(), false) + } +} + +fn is_json_mime(mime: &Mime) -> bool { + if mime.type_() == mime::APPLICATION && mime.subtype() == mime::JSON { + return true; + } + + if let Some(suffix) = mime.suffix() { + if suffix == mime::JSON { + return true; } - Value::String(s) => s.clone(), - Value::Number(n) => n.to_string(), - Value::Bool(b) => b.to_string(), - Value::Null => "null".to_string(), } + + mime.subtype().as_str().ends_with("+json") } + +fn is_text_mime(mime: &Mime) -> bool { + if mime.type_() == mime::TEXT { + return true; + } + + if let Some(suffix) = mime.suffix() { + if matches!(suffix, mime::XML | mime::JSON) { + return true; + } + } + + matches!( + mime.subtype().as_str(), + "xml" | "json" | "javascript" | "x-javascript" | "x-www-form-urlencoded" | "csv" | "plain" | "html" + ) +} + +fn looks_like_json(body: &[u8]) -> bool { + body + .iter() + .copied() + .skip_while(|b| b.is_ascii_whitespace()) + .next() + .map_or(false, |b| matches!(b, b'{' | b'[')) +} + bindings::export!(Component with_types_in bindings);