From a21bd46952bcb5c8bc5eff79ea09a6acdd046d24 Mon Sep 17 00:00:00 2001 From: Dmitry Ivankov Date: Wed, 4 Dec 2024 16:32:13 +0100 Subject: [PATCH] Stream file downloads --- build.sbt | 4 +- build.scala-2.12.sbt.lock | 258 ++---------------- build.scala-2.13.sbt.lock | 258 ++---------------- build.scala-3.sbt.lock | 258 ++---------------- .../sdk/scala/v1/resources/files.scala | 13 +- .../com/cognite/sdk/scala/v1/FilesTest.scala | 9 +- 6 files changed, 95 insertions(+), 705 deletions(-) diff --git a/build.sbt b/build.sbt index b8ca0f8eb..82f7cd31a 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ lazy val commonSettings = Seq( organization := "com.cognite", organizationName := "Cognite", organizationHomepage := Some(url("https://cognite.com")), - version := "2.31." + patchVersion, + version := "2.32." + patchVersion, isSnapshot := patchVersion.endsWith("-SNAPSHOT"), scalaVersion := scala213, // use 2.13 by default // handle cross plugin https://github.com/stringbean/sbt-dependency-lock/issues/13 @@ -192,7 +192,7 @@ val sttpDeps = Seq( .exclude("io.circe", "circe-parser_2.11") .exclude("io.circe", "circe-parser_2.12") .exclude("io.circe", "circe-parser_2.13"), - "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats" % sttpVersion % Test + "com.softwaremill.sttp.client3" %% "fs2" % sttpVersion ) def circeDeps(scalaVersion: Option[(Long, Long)]): Seq[ModuleID] = diff --git a/build.scala-2.12.sbt.lock b/build.scala-2.12.sbt.lock index 6ea9e342d..1b9a6c70d 100644 --- a/build.scala-2.12.sbt.lock +++ b/build.scala-2.12.sbt.lock @@ -1,6 +1,6 @@ { "lockVersion" : 1, - "timestamp" : "2024-12-01T21:11:38.711996Z", + "timestamp" : "2024-12-04T15:21:57.209479534Z", "configurations" : [ "compile", "optional", @@ -93,54 +93,44 @@ }, { "org" : "com.softwaremill.sttp.client3", - "name" : "async-http-client-backend-cats_2.12", - "version" : "3.5.2", - "artifacts" : [ - { - "name" : "async-http-client-backend-cats_2.12.jar", - "hash" : "sha1:c32dd8df1ede62e710312266a57b26fae7bf8b86" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "com.softwaremill.sttp.client3", - "name" : "async-http-client-backend_2.12", + "name" : "cats_2.12", "version" : "3.5.2", "artifacts" : [ { - "name" : "async-http-client-backend_2.12.jar", - "hash" : "sha1:59b7740ed98e783b3b6bfd096edefaa0b7d458ec" + "name" : "cats_2.12.jar", + "hash" : "sha1:a859ae377bfdbb1d7e16f58890b84d163f12d646" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, { "org" : "com.softwaremill.sttp.client3", - "name" : "cats_2.12", + "name" : "circe_2.12", "version" : "3.5.2", "artifacts" : [ { - "name" : "cats_2.12.jar", - "hash" : "sha1:a859ae377bfdbb1d7e16f58890b84d163f12d646" + "name" : "circe_2.12.jar", + "hash" : "sha1:6e7629e4472a797186f0df00f339f5f708dbca8d" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, { "org" : "com.softwaremill.sttp.client3", - "name" : "circe_2.12", + "name" : "core_2.12", "version" : "3.5.2", "artifacts" : [ { - "name" : "circe_2.12.jar", - "hash" : "sha1:6e7629e4472a797186f0df00f339f5f708dbca8d" + "name" : "core_2.12.jar", + "hash" : "sha1:999f1ed7549feadcf0fac88cbbd39cd43b8d51bc" } ], "configurations" : [ @@ -151,12 +141,12 @@ }, { "org" : "com.softwaremill.sttp.client3", - "name" : "core_2.12", + "name" : "fs2_2.12", "version" : "3.5.2", "artifacts" : [ { - "name" : "core_2.12.jar", - "hash" : "sha1:999f1ed7549feadcf0fac88cbbd39cd43b8d51bc" + "name" : "fs2_2.12.jar", + "hash" : "sha1:8dde4d364d013efd245e5d4b849ff3b1e202d5f2" } ], "configurations" : [ @@ -215,12 +205,12 @@ }, { "org" : "com.softwaremill.sttp.shared", - "name" : "ws_2.12", + "name" : "fs2_2.12", "version" : "1.3.4", "artifacts" : [ { - "name" : "ws_2.12.jar", - "hash" : "sha1:60650da91689e95760cd837bb13732dc94fe6d18" + "name" : "fs2_2.12.jar", + "hash" : "sha1:a53a923369b494da9283bd0bb9c34639f9d63f8a" } ], "configurations" : [ @@ -230,30 +220,18 @@ ] }, { - "org" : "com.sun.activation", - "name" : "jakarta.activation", - "version" : "1.2.2", - "artifacts" : [ - { - "name" : "jakarta.activation.jar", - "hash" : "sha1:74548703f9851017ce2f556066659438019e7eb5" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "com.typesafe.netty", - "name" : "netty-reactive-streams", - "version" : "2.0.4", + "org" : "com.softwaremill.sttp.shared", + "name" : "ws_2.12", + "version" : "1.3.4", "artifacts" : [ { - "name" : "netty-reactive-streams.jar", - "hash" : "sha1:0f77c8eaa7d5e2f2160b6d21ba385cf726f164b2" + "name" : "ws_2.12.jar", + "hash" : "sha1:60650da91689e95760cd837bb13732dc94fe6d18" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, @@ -367,132 +345,6 @@ "test" ] }, - { - "org" : "io.netty", - "name" : "netty-buffer", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-buffer.jar", - "hash" : "sha1:9d213d090deeca2541ad6827eb3345bcd6e1e701" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec.jar", - "hash" : "sha1:b740d51babe3312a33b505cd8b62c02f3731b2e3" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec-http", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec-http.jar", - "hash" : "sha1:354ca712f909fcd9d8a22e16bf84afcfdc44ec8d" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec-socks", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec-socks.jar", - "hash" : "sha1:6f4573281df659265bd709fd10471c3e00ef6c70" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-common", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-common.jar", - "hash" : "sha1:44540113f7148f1014be879663501db8da1c37b0" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-handler", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-handler.jar", - "hash" : "sha1:14e28bab0173be10c9631a85069636a0d0221dfe" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-handler-proxy", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-handler-proxy.jar", - "hash" : "sha1:2352f12826400e5db64b36fd951508ce9a61c196" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-resolver", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-resolver.jar", - "hash" : "sha1:caba5004618d27386ee9d5ee8b23b09b6548fb0b" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-transport", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-transport.jar", - "hash" : "sha1:94350c81cc7a78212fb0f52a500f22d1aa9c44d8" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "javax.servlet", "name" : "javax.servlet-api", @@ -507,34 +359,6 @@ "test" ] }, - { - "org" : "org.asynchttpclient", - "name" : "async-http-client", - "version" : "2.12.3", - "artifacts" : [ - { - "name" : "async-http-client.jar", - "hash" : "sha1:6dfc91814cc8b3bc3327246d0e5df36911b9a623" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "org.asynchttpclient", - "name" : "async-http-client-netty-utils", - "version" : "2.12.3", - "artifacts" : [ - { - "name" : "async-http-client-netty-utils.jar", - "hash" : "sha1:ad99d8622931ed31367d0fef7fa17eb62e033fb3" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.eclipse.jetty", "name" : "jetty-http", @@ -633,20 +457,6 @@ "test" ] }, - { - "org" : "org.reactivestreams", - "name" : "reactive-streams", - "version" : "1.0.3", - "artifacts" : [ - { - "name" : "reactive-streams.jar", - "hash" : "sha1:d9fb7a7926ffa635b3dcaa5049fb2bfa25b3e7d0" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.scala-lang", "name" : "scala-library", @@ -975,20 +785,6 @@ "test" ] }, - { - "org" : "org.slf4j", - "name" : "slf4j-api", - "version" : "1.7.30", - "artifacts" : [ - { - "name" : "slf4j-api.jar", - "hash" : "sha1:b5a4b6d16ab13e34a88fae84c35cd5d68cac922c" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.tpolecat", "name" : "natchez-core_2.12", diff --git a/build.scala-2.13.sbt.lock b/build.scala-2.13.sbt.lock index 49c7589f6..4e573bbdc 100644 --- a/build.scala-2.13.sbt.lock +++ b/build.scala-2.13.sbt.lock @@ -1,6 +1,6 @@ { "lockVersion" : 1, - "timestamp" : "2024-12-01T21:11:41.065267Z", + "timestamp" : "2024-12-04T15:21:58.634889123Z", "configurations" : [ "compile", "optional", @@ -93,54 +93,44 @@ }, { "org" : "com.softwaremill.sttp.client3", - "name" : "async-http-client-backend-cats_2.13", - "version" : "3.5.2", - "artifacts" : [ - { - "name" : "async-http-client-backend-cats_2.13.jar", - "hash" : "sha1:de5a144cbc65590655124a03122e531921ee4b59" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "com.softwaremill.sttp.client3", - "name" : "async-http-client-backend_2.13", + "name" : "cats_2.13", "version" : "3.5.2", "artifacts" : [ { - "name" : "async-http-client-backend_2.13.jar", - "hash" : "sha1:88cbf4ffda294827e176a407e1f9b27f2ad5beb6" + "name" : "cats_2.13.jar", + "hash" : "sha1:d19b881660ae9876681a5f30b43e53ee97847ff4" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, { "org" : "com.softwaremill.sttp.client3", - "name" : "cats_2.13", + "name" : "circe_2.13", "version" : "3.5.2", "artifacts" : [ { - "name" : "cats_2.13.jar", - "hash" : "sha1:d19b881660ae9876681a5f30b43e53ee97847ff4" + "name" : "circe_2.13.jar", + "hash" : "sha1:235cd40423f2e97bbcb250234f8991f34d98ad0e" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, { "org" : "com.softwaremill.sttp.client3", - "name" : "circe_2.13", + "name" : "core_2.13", "version" : "3.5.2", "artifacts" : [ { - "name" : "circe_2.13.jar", - "hash" : "sha1:235cd40423f2e97bbcb250234f8991f34d98ad0e" + "name" : "core_2.13.jar", + "hash" : "sha1:c40e81feaafdec5f8d1861a9298d9c4dd755d25e" } ], "configurations" : [ @@ -151,12 +141,12 @@ }, { "org" : "com.softwaremill.sttp.client3", - "name" : "core_2.13", + "name" : "fs2_2.13", "version" : "3.5.2", "artifacts" : [ { - "name" : "core_2.13.jar", - "hash" : "sha1:c40e81feaafdec5f8d1861a9298d9c4dd755d25e" + "name" : "fs2_2.13.jar", + "hash" : "sha1:f169cced1665841097e00fe43e34f206fc854d8f" } ], "configurations" : [ @@ -215,12 +205,12 @@ }, { "org" : "com.softwaremill.sttp.shared", - "name" : "ws_2.13", + "name" : "fs2_2.13", "version" : "1.3.4", "artifacts" : [ { - "name" : "ws_2.13.jar", - "hash" : "sha1:7afdc7333ed514b5fc7822509909669b681cb162" + "name" : "fs2_2.13.jar", + "hash" : "sha1:242af688949c9e2469df3e2ac5628dec24e761c0" } ], "configurations" : [ @@ -230,30 +220,18 @@ ] }, { - "org" : "com.sun.activation", - "name" : "jakarta.activation", - "version" : "1.2.2", - "artifacts" : [ - { - "name" : "jakarta.activation.jar", - "hash" : "sha1:74548703f9851017ce2f556066659438019e7eb5" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "com.typesafe.netty", - "name" : "netty-reactive-streams", - "version" : "2.0.4", + "org" : "com.softwaremill.sttp.shared", + "name" : "ws_2.13", + "version" : "1.3.4", "artifacts" : [ { - "name" : "netty-reactive-streams.jar", - "hash" : "sha1:0f77c8eaa7d5e2f2160b6d21ba385cf726f164b2" + "name" : "ws_2.13.jar", + "hash" : "sha1:7afdc7333ed514b5fc7822509909669b681cb162" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, @@ -367,132 +345,6 @@ "test" ] }, - { - "org" : "io.netty", - "name" : "netty-buffer", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-buffer.jar", - "hash" : "sha1:9d213d090deeca2541ad6827eb3345bcd6e1e701" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec.jar", - "hash" : "sha1:b740d51babe3312a33b505cd8b62c02f3731b2e3" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec-http", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec-http.jar", - "hash" : "sha1:354ca712f909fcd9d8a22e16bf84afcfdc44ec8d" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec-socks", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec-socks.jar", - "hash" : "sha1:6f4573281df659265bd709fd10471c3e00ef6c70" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-common", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-common.jar", - "hash" : "sha1:44540113f7148f1014be879663501db8da1c37b0" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-handler", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-handler.jar", - "hash" : "sha1:14e28bab0173be10c9631a85069636a0d0221dfe" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-handler-proxy", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-handler-proxy.jar", - "hash" : "sha1:2352f12826400e5db64b36fd951508ce9a61c196" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-resolver", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-resolver.jar", - "hash" : "sha1:caba5004618d27386ee9d5ee8b23b09b6548fb0b" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-transport", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-transport.jar", - "hash" : "sha1:94350c81cc7a78212fb0f52a500f22d1aa9c44d8" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "javax.servlet", "name" : "javax.servlet-api", @@ -507,34 +359,6 @@ "test" ] }, - { - "org" : "org.asynchttpclient", - "name" : "async-http-client", - "version" : "2.12.3", - "artifacts" : [ - { - "name" : "async-http-client.jar", - "hash" : "sha1:6dfc91814cc8b3bc3327246d0e5df36911b9a623" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "org.asynchttpclient", - "name" : "async-http-client-netty-utils", - "version" : "2.12.3", - "artifacts" : [ - { - "name" : "async-http-client-netty-utils.jar", - "hash" : "sha1:ad99d8622931ed31367d0fef7fa17eb62e033fb3" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.eclipse.jetty", "name" : "jetty-http", @@ -633,20 +457,6 @@ "test" ] }, - { - "org" : "org.reactivestreams", - "name" : "reactive-streams", - "version" : "1.0.3", - "artifacts" : [ - { - "name" : "reactive-streams.jar", - "hash" : "sha1:d9fb7a7926ffa635b3dcaa5049fb2bfa25b3e7d0" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.scala-lang", "name" : "scala-library", @@ -975,20 +785,6 @@ "test" ] }, - { - "org" : "org.slf4j", - "name" : "slf4j-api", - "version" : "1.7.30", - "artifacts" : [ - { - "name" : "slf4j-api.jar", - "hash" : "sha1:b5a4b6d16ab13e34a88fae84c35cd5d68cac922c" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.tpolecat", "name" : "natchez-core_2.13", diff --git a/build.scala-3.sbt.lock b/build.scala-3.sbt.lock index a031cd3be..b4889b3d7 100644 --- a/build.scala-3.sbt.lock +++ b/build.scala-3.sbt.lock @@ -1,6 +1,6 @@ { "lockVersion" : 1, - "timestamp" : "2024-12-01T21:11:43.214140Z", + "timestamp" : "2024-12-04T15:22:01.176889760Z", "configurations" : [ "compile", "optional", @@ -77,54 +77,44 @@ }, { "org" : "com.softwaremill.sttp.client3", - "name" : "async-http-client-backend-cats_3", - "version" : "3.5.2", - "artifacts" : [ - { - "name" : "async-http-client-backend-cats_3.jar", - "hash" : "sha1:fa2d0e379281c7c03535d3f76e385df7708458f5" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "com.softwaremill.sttp.client3", - "name" : "async-http-client-backend_3", + "name" : "cats_3", "version" : "3.5.2", "artifacts" : [ { - "name" : "async-http-client-backend_3.jar", - "hash" : "sha1:77b83fd573c73d245786bd6de062aa61fa205bbf" + "name" : "cats_3.jar", + "hash" : "sha1:3a32c66453ccf274d7c9affd7402de2c75e4ef01" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, { "org" : "com.softwaremill.sttp.client3", - "name" : "cats_3", + "name" : "circe_3", "version" : "3.5.2", "artifacts" : [ { - "name" : "cats_3.jar", - "hash" : "sha1:3a32c66453ccf274d7c9affd7402de2c75e4ef01" + "name" : "circe_3.jar", + "hash" : "sha1:ae220994383a40788148acaa2ec907a46df8cd7a" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, { "org" : "com.softwaremill.sttp.client3", - "name" : "circe_3", + "name" : "core_3", "version" : "3.5.2", "artifacts" : [ { - "name" : "circe_3.jar", - "hash" : "sha1:ae220994383a40788148acaa2ec907a46df8cd7a" + "name" : "core_3.jar", + "hash" : "sha1:30f9d5bda8b0073caa20e8e9ef21fbc93ba3ad9f" } ], "configurations" : [ @@ -135,12 +125,12 @@ }, { "org" : "com.softwaremill.sttp.client3", - "name" : "core_3", + "name" : "fs2_3", "version" : "3.5.2", "artifacts" : [ { - "name" : "core_3.jar", - "hash" : "sha1:30f9d5bda8b0073caa20e8e9ef21fbc93ba3ad9f" + "name" : "fs2_3.jar", + "hash" : "sha1:0f6a39440de5e807580e16a5ca42c5348109171c" } ], "configurations" : [ @@ -199,12 +189,12 @@ }, { "org" : "com.softwaremill.sttp.shared", - "name" : "ws_3", + "name" : "fs2_3", "version" : "1.3.4", "artifacts" : [ { - "name" : "ws_3.jar", - "hash" : "sha1:4b6842fa1b487d34b6e8dfc529a9f7c3b402339b" + "name" : "fs2_3.jar", + "hash" : "sha1:6de325c0e8b0aba3d12607a97b77e6b550c7bdc0" } ], "configurations" : [ @@ -214,30 +204,18 @@ ] }, { - "org" : "com.sun.activation", - "name" : "jakarta.activation", - "version" : "1.2.2", - "artifacts" : [ - { - "name" : "jakarta.activation.jar", - "hash" : "sha1:74548703f9851017ce2f556066659438019e7eb5" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "com.typesafe.netty", - "name" : "netty-reactive-streams", - "version" : "2.0.4", + "org" : "com.softwaremill.sttp.shared", + "name" : "ws_3", + "version" : "1.3.4", "artifacts" : [ { - "name" : "netty-reactive-streams.jar", - "hash" : "sha1:0f77c8eaa7d5e2f2160b6d21ba385cf726f164b2" + "name" : "ws_3.jar", + "hash" : "sha1:4b6842fa1b487d34b6e8dfc529a9f7c3b402339b" } ], "configurations" : [ + "compile", + "runtime", "test" ] }, @@ -351,132 +329,6 @@ "test" ] }, - { - "org" : "io.netty", - "name" : "netty-buffer", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-buffer.jar", - "hash" : "sha1:9d213d090deeca2541ad6827eb3345bcd6e1e701" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec.jar", - "hash" : "sha1:b740d51babe3312a33b505cd8b62c02f3731b2e3" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec-http", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec-http.jar", - "hash" : "sha1:354ca712f909fcd9d8a22e16bf84afcfdc44ec8d" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-codec-socks", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-codec-socks.jar", - "hash" : "sha1:6f4573281df659265bd709fd10471c3e00ef6c70" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-common", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-common.jar", - "hash" : "sha1:44540113f7148f1014be879663501db8da1c37b0" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-handler", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-handler.jar", - "hash" : "sha1:14e28bab0173be10c9631a85069636a0d0221dfe" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-handler-proxy", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-handler-proxy.jar", - "hash" : "sha1:2352f12826400e5db64b36fd951508ce9a61c196" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-resolver", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-resolver.jar", - "hash" : "sha1:caba5004618d27386ee9d5ee8b23b09b6548fb0b" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "io.netty", - "name" : "netty-transport", - "version" : "4.1.60.Final", - "artifacts" : [ - { - "name" : "netty-transport.jar", - "hash" : "sha1:94350c81cc7a78212fb0f52a500f22d1aa9c44d8" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "javax.servlet", "name" : "javax.servlet-api", @@ -491,34 +343,6 @@ "test" ] }, - { - "org" : "org.asynchttpclient", - "name" : "async-http-client", - "version" : "2.12.3", - "artifacts" : [ - { - "name" : "async-http-client.jar", - "hash" : "sha1:6dfc91814cc8b3bc3327246d0e5df36911b9a623" - } - ], - "configurations" : [ - "test" - ] - }, - { - "org" : "org.asynchttpclient", - "name" : "async-http-client-netty-utils", - "version" : "2.12.3", - "artifacts" : [ - { - "name" : "async-http-client-netty-utils.jar", - "hash" : "sha1:ad99d8622931ed31367d0fef7fa17eb62e033fb3" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.eclipse.jetty", "name" : "jetty-http", @@ -617,20 +441,6 @@ "test" ] }, - { - "org" : "org.reactivestreams", - "name" : "reactive-streams", - "version" : "1.0.3", - "artifacts" : [ - { - "name" : "reactive-streams.jar", - "hash" : "sha1:d9fb7a7926ffa635b3dcaa5049fb2bfa25b3e7d0" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.scala-lang", "name" : "scala-library", @@ -961,20 +771,6 @@ "test" ] }, - { - "org" : "org.slf4j", - "name" : "slf4j-api", - "version" : "1.7.30", - "artifacts" : [ - { - "name" : "slf4j-api.jar", - "hash" : "sha1:b5a4b6d16ab13e34a88fae84c35cd5d68cac922c" - } - ], - "configurations" : [ - "test" - ] - }, { "org" : "org.tpolecat", "name" : "natchez-core_3", diff --git a/src/main/scala/com/cognite/sdk/scala/v1/resources/files.scala b/src/main/scala/com/cognite/sdk/scala/v1/resources/files.scala index 27087b86d..dbd02253a 100644 --- a/src/main/scala/com/cognite/sdk/scala/v1/resources/files.scala +++ b/src/main/scala/com/cognite/sdk/scala/v1/resources/files.scala @@ -4,15 +4,17 @@ package com.cognite.sdk.scala.v1.resources import java.io.{BufferedInputStream, FileInputStream} - import cats.implicits._ import cats.effect.Sync import com.cognite.sdk.scala.common._ import com.cognite.sdk.scala.v1._ +import fs2.Stream import sttp.client3._ import sttp.client3.circe._ import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import sttp.capabilities.fs2.Fs2Streams +import sttp.client3.internal.NoStreams.BinaryStream class Files[F[_]](val requestSession: RequestSession[F]) extends WithRequestSession[F] @@ -148,22 +150,21 @@ class Files[F[_]](val requestSession: RequestSession[F]) ) ) - def download(item: FileDownload, out: java.io.OutputStream)(implicit F: Sync[F]): F[Unit] = + def download(item: FileDownload): F[Stream[F, Byte]] = for { link <- downloadLink(item) res <- requestSession.send { request => request .get(uri"${link.downloadUrl}") - .response(asByteArray) + .response(asStreamUnsafe(Fs2Streams[F])) } - bytes <- F.fromOption( + bytesStream <- F.fromOption( res.body.toOption, SdkException( s"File download of file ${item.toString} failed with error code ${res.code.toString}" ) ) - r <- F.blocking(out.write(bytes)) - } yield r + } yield bytesStream } object Files { diff --git a/src/test/scala/com/cognite/sdk/scala/v1/FilesTest.scala b/src/test/scala/com/cognite/sdk/scala/v1/FilesTest.scala index d09af0481..b02a20d01 100644 --- a/src/test/scala/com/cognite/sdk/scala/v1/FilesTest.scala +++ b/src/test/scala/com/cognite/sdk/scala/v1/FilesTest.scala @@ -8,7 +8,7 @@ import java.nio.file.{Files, Paths} import java.time.Instant import java.util.UUID import com.cognite.sdk.scala.common.{CdpApiException, ReadBehaviours, RetryWhile, SdkTestSpec, SetValue, WritableBehaviors} -import fs2.Stream +import fs2.{Collector, Stream} import org.scalatest.matchers.should.Matchers @SuppressWarnings( @@ -289,13 +289,14 @@ class FilesTest extends SdkTestSpec with ReadBehaviours with WritableBehaviors w } } - val out = new ByteArrayOutputStream() - client.files.download(FileDownloadId(file.id), out).unsafeRunSync() + val out = client.files.download(FileDownloadId(file.id)).flatMap(_.compile.toVector) + .map(_.toArray) + .unsafeRunSync() val expected = Files.readAllBytes(Paths.get("./src/test/scala/com/cognite/sdk/scala/v1/uploadTest.txt")) - assert(out.toByteArray.sameElements(expected)) + assert(out.sameElements(expected)) client.files.deleteById(file.id).unsafeRunSync() }