diff --git a/.travis.yml b/.travis.yml index c25655031..537f317c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,8 @@ install: - sudo apt-get install -y rpm language: scala -jdk: openjdk8 +# java 11+ needed for sttp +jdk: openjdk21 env: global: diff --git a/admin/build.ps1 b/admin/build.ps1 index 0d7836b80..0ba8d771e 100644 --- a/admin/build.ps1 +++ b/admin/build.ps1 @@ -34,9 +34,12 @@ if ($env:APPVEYOR_FORCED_BUILD -eq 'true') { ensureVersion clearIvyCache if ($env:mode -eq 'release') { - echo "Running a release for $env:version" - $repositoriesFile="$env:APPVEYOR_BUILD_FOLDER\conf\repositories" - & cmd /c "sbt ""-Dsbt.override.build.repos=true"" ""-Dsbt.repository.config=$repositoriesFile"" ""-Dproject.version=$env:version"" ""show fullResolvers"" clean update s3Upload" '2>&1' + if ($env:version -match '-bin-' -or $env:version -match '-pre-') { + & cmd /c "sbt ""-Dproject.version=$env:version"" clean update ""show s3Upload/mappings""" '2>&1' + } else { + echo "Running a release for $env:version" + & cmd /c "sbt ""-Dproject.version=$env:version"" clean update ghUpload" '2>&1' + } checkExit } else { echo "Unknown mode: '$env:mode'" diff --git a/admin/build.sh b/admin/build.sh index a2ac82ce5..6c8ba985d 100755 --- a/admin/build.sh +++ b/admin/build.sh @@ -90,15 +90,20 @@ if [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then setupSSH . scripts/jobs/release/website/update-api elif [[ "$mode" == "release" ]]; then - echo "Running a release for $version" triggerMsiRelease - repositoriesFile="$TRAVIS_BUILD_DIR/conf/repositories" - # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version - sbt -Dsbt.log.noformat=true \ - -Dsbt.override.build.repos=true -Dsbt.repository.config="$repositoriesFile" \ - -Dproject.version=$version \ - "show fullResolvers" clean update s3Upload - triggerSmoketest + if [[ "$version" =~ -bin- || "$version" =~ -pre- ]]; then + # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version + sbt -Dsbt.log.noformat=true \ + -Dproject.version=$version \ + clean update "show s3Upload/mappings" + else + echo "Running a release for $version" + # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version + sbt -Dsbt.log.noformat=true \ + -Dproject.version=$version \ + clean update ghUpload + triggerSmoketest + fi else echo "Unknown build mode: '$mode'" exit 1 diff --git a/appveyor.yml b/appveyor.yml index 56fc955e6..80fd54566 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,10 +6,11 @@ branches: - 2.12.x - 2.13.x -image: Visual Studio 2015 +image: Visual Studio 2019 environment: - JAVA_HOME: C:\Program Files\Java\jdk1.8.0 + # java 11+ needed for sttp + JAVA_HOME: C:\Program Files\Java\jdk21 AWS_ACCESS_KEY_ID: secure: X1Ix1soRBDMtfbi8IFNPOggDP2XquhW+uKcJ+XC0kiM= AWS_SECRET_ACCESS_KEY: @@ -18,7 +19,7 @@ environment: secure: nfWNfbyzNQwdg1eWHZX93XIJLoAhFOr1fR8+L86s7a3jdo/HydEZ8TyEKsPipQ+/ install: - - cmd: choco install sbt --version=1.3.2 -ia "INSTALLDIR=""C:\sbt""" + - cmd: choco install sbt --version=1.10.11 -ia "INSTALLDIR=""C:\sbt""" - cmd: SET PATH=C:\sbt\bin;%JAVA_HOME%\bin;%PATH% build_script: diff --git a/build.sbt b/build.sbt index 4ca47182e..28f0c1995 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,6 @@ -import ScalaDist.upload +import ScalaDist.{s3Upload, ghUpload} + +resolvers += "scala-integration" at "https://scala-ci.typesafe.com/artifactory/scala-integration/" // so we don't require a native git install useJGit @@ -22,9 +24,9 @@ Versioning.settings // are known/understood, at scala/scala-dist#171 scalaVersion := version.value -upload / mappings := Seq() +s3Upload / mappings := Seq() -upload := { +s3Upload := { import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.amazonaws.services.s3.model.PutObjectRequest import com.amazonaws.regions.Regions @@ -33,12 +35,50 @@ upload := { val client = AmazonS3ClientBuilder.standard.withRegion(Regions.US_EAST_1).build val log = streams.value.log - (upload / mappings).value map { case (file, key) => + (s3Upload / mappings).value map { case (file, key) => log.info("Uploading "+ file.getAbsolutePath() +" as "+ key) client.putObject(new PutObjectRequest("downloads.typesafe.com", key, file)) } } +ghUpload := { + import sttp.client3._ + import _root_.io.circe._, _root_.io.circe.parser._ + + val log = streams.value.log + val ghRelease = s"v${(Universal / version).value}" + + val token = sys.env.getOrElse("GITHUB_OAUTH_TOKEN", throw new MessageOnlyException("GITHUB_OAUTH_TOKEN missing")) + + val backend = HttpURLConnectionBackend() + + val rRes = basicRequest + .get(uri"https://api.github.com/repos/scala/scala/releases/tags/$ghRelease") + .header("Accept", "application/vnd.github+json") + .header("Authorization", s"Bearer $token") + .header("X-GitHub-Api-Version", "2022-11-28") + .send(backend) + val releaseId = rRes.body.flatMap(parse).getOrElse(Json.Null).hcursor.downField("id").as[Int].getOrElse( + throw new MessageOnlyException(s"Release not found: $ghRelease")) + + (s3Upload / mappings).value map { case (file, _) => + log.info(s"Uploading ${file.getAbsolutePath} as ${file.getName} to https://github.com/scala/scala/releases/tag/$ghRelease") + + // https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset + val request = basicRequest + .post(uri"https://uploads.github.com/repos/scala/scala/releases/${releaseId}/assets?name=${file.getName}") + .contentType("application/octet-stream") + .header("Accept", "application/vnd.github+json") + .header("Authorization", s"Bearer $token") + .header("X-GitHub-Api-Version", "2022-11-28") + .body(file) + + val response = request.send(backend) + if (response.code.code != 201) + throw new MessageOnlyException(s"Upload failed: status=${response.code}\n${response.body}") + } +} + ScalaDist.settings Docs.settings diff --git a/conf/repositories b/conf/repositories deleted file mode 100644 index 3e92a67b0..000000000 --- a/conf/repositories +++ /dev/null @@ -1,8 +0,0 @@ -[repositories] - scala-integration: https://scala-ci.typesafe.com/artifactory/scala-integration/ - sonatype-staging: https://oss.sonatype.org/content/repositories/staging/ - jcenter-cache: https://scala-ci.typesafe.com/artifactory/jcenter/ - typesafe-ivy-releases: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly - sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] - maven-central - local diff --git a/project/ScalaDist.scala b/project/ScalaDist.scala index 58d402fcb..a6d2872ed 100644 --- a/project/ScalaDist.scala +++ b/project/ScalaDist.scala @@ -10,7 +10,8 @@ import com.amazonaws.services.s3.model.PutObjectResult // can't call it Universal -- that's taken by the packager object ScalaDist { - val upload=TaskKey[Seq[PutObjectResult]]("s3-upload","Uploads files to an S3 bucket.") + val s3Upload = TaskKey[Seq[PutObjectResult]]("s3Upload","Uploads files to an S3 bucket.") + val ghUpload = TaskKey[Seq[Unit]]("ghUpload","Uploads files to GitHub Releases.") def createMappingsWith(deps: Seq[(sbt.librarymanagement.ConfigRef, ModuleID, Artifact, File)], distMappingGen: (ModuleID, Artifact, File) => Seq[(File, String)]): Seq[(File, String)] = @@ -21,30 +22,30 @@ object ScalaDist { case _ => Seq() } - // used to make s3-upload upload the file produced by fileTask to the path scala/$version/${file.name} + // used to make s3Upload upload the file produced by fileTask to the path scala/$version/${file.name} private def uploadMapping(fileTask: TaskKey[File]) = Def.task { val file = fileTask.value file -> s"scala/${version.value}/${file.getName}" } - // make it so that s3-upload will upload the msi when we're running on windows, and everything else when we're on linux - // s3-upload thus depends on the package tasks listed below + // make it so that s3Upload will upload the msi when we're running on windows, and everything else when we're on linux + // s3Upload thus depends on the package tasks listed below def platformSettings = if (sys.props("os.name").toLowerCase(java.util.Locale.US) contains "windows") - Wix.settings :+ (upload / mappings += uploadMapping(Windows / packageBin).value) + Wix.settings :+ (s3Upload / mappings += uploadMapping(Windows / packageBin).value) else Unix.settings ++ Seq( - upload / mappings += uploadMapping(Universal / packageBin).value, - upload / mappings += uploadMapping(Universal / packageZipTarball).value, - upload / mappings += uploadMapping(UniversalDocs / packageBin).value, - upload / mappings += uploadMapping(UniversalDocs / packageZipTarball).value, - upload / mappings += uploadMapping(UniversalDocs / packageXzTarball).value, - upload / mappings += uploadMapping(Rpm / packageBin).value, + s3Upload / mappings += uploadMapping(Universal / packageBin).value, + s3Upload / mappings += uploadMapping(Universal / packageZipTarball).value, + s3Upload / mappings += uploadMapping(UniversalDocs / packageBin).value, + s3Upload / mappings += uploadMapping(UniversalDocs / packageZipTarball).value, + s3Upload / mappings += uploadMapping(UniversalDocs / packageXzTarball).value, + s3Upload / mappings += uploadMapping(Rpm / packageBin).value, // Debian needs special handling because the value sbt-native-packager // gives us for `Debian / packageBin` (coming from the archiveFilename // method) includes the debian version and arch information, // which we historically have not included. I don't see a way to // override the filename on disk, so we re-map at upload time - upload / mappings += Def.task { + s3Upload / mappings += Def.task { (Debian / packageBin).value -> s"scala/${version.value}/${(Debian / name).value}-${version.value}.deb" }.value @@ -65,13 +66,6 @@ object ScalaDist { // create lib directory by resolving scala-dist's dependencies // to populate the rest of the distribution, explode scala-dist artifact itself Universal / mappings ++= createMappingsWith(update.value.toSeq, universalMappings), - - // work around regression in sbt-native-packager 1.0.5 where - // these tasks invoke `tar` without any flags at all. the issue - // was fixed in 1.1.0, so this could be revisited when we upgrade - UniversalDocs / packageZipTarball / universalArchiveOptions := Seq("--force-local", "-pcvf"), - UniversalDocs / packageXzTarball / universalArchiveOptions := Seq("--force-local", "-pcvf") - ) // private lazy val onWindows = System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows") diff --git a/project/build.properties b/project/build.properties index e88a0d817..cc68b53f1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.6 +sbt.version=1.10.11 diff --git a/project/plugins.sbt b/project/plugins.sbt index 1048fe1e3..76d716f14 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -7,5 +7,11 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.8.1") libraryDependencies += "com.amazonaws" % "aws-java-sdk-s3" % "1.12.5" -addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.1") +libraryDependencies += "com.softwaremill.sttp.client3" %% "core" % "3.11.0" +libraryDependencies ++= Seq( + "io.circe" %% "circe-core", + "io.circe" %% "circe-generic", + "io.circe" %% "circe-parser" +).map(_ % "0.14.13") +addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.1") diff --git a/scripts/common b/scripts/common index 782bd0243..0832adc91 100755 --- a/scripts/common +++ b/scripts/common @@ -14,7 +14,7 @@ function travis_fold_end() { function postCommitStatus() { if [[ "$scala_sha" != "" ]]; then local jsonTemplate='{ "state": "%s", "target_url": "%s", "description": "%s", "context": "%s"}' - local json=$(printf "$jsonTemplate" "$1" "https://travis-ci.com/scala/scala-dist/builds/$TRAVIS_BUILD_ID" "$1" "travis/scala-dist/$version/$mode") + local json=$(printf "$jsonTemplate" "$1" "https://app.travis-ci.com/github/scala/scala-dist/builds/$TRAVIS_BUILD_ID" "$1" "travis/scala-dist/$version/$mode") [[ -z "$GITHUB_OAUTH_TOKEN" ]] && (echo "Missing environment variable GITHUB_OAUTH_TOKEN!"; exit 1) TMPFILE=$(mktemp -t curl.XXXXXXXXXX) diff --git a/scripts/jobs/release/package/generic b/scripts/jobs/release/package/generic deleted file mode 100755 index a80decc6c..000000000 --- a/scripts/jobs/release/package/generic +++ /dev/null @@ -1,32 +0,0 @@ -# @pre current directory == repo root -# must run on both windows (cygwin) and linux - -# invoked from scala-release-2.13.x-[unix|windows] - -# uses the following env vars: -# - WORKSPACE, JAVA_OPTS (jenkins standard) -# - version checked for consistency against HEAD's tag -# - sbtLauncher is a global jenkins configuration variable -# - sbtDistTarget the target (beyond clean and update) to run -# - sbtDistVersionOverride may be -Dproject.version=.... to override the version of the dist to build (only for testing, you should use tags for real releases!) - -# requires git (>= 1.8), java, bash - -repositoriesFile="$WORKSPACE/conf/repositories" - -tags="$(git tag --points-at HEAD)" - -# version is set by the scala-release-2.13.x-dist build flow, make sure it's consistent with the tag -# this is also a backstop for https://github.com/sbt/sbt-git/issues/35 -# ignore when there is no version (job is running outside of the flow) -[[ -z $sbtDistVersionOverride ]] && [[ -n $version ]] && if [ "$tags" != "v$version" ]; then - echo "Inconsistent tag/version combo detected. Abort-abort." - exit 255 -fi - -# want full control over sbt, so invoke the launcher directly -java $JAVA_OPTS -Dsbt.log.noformat=true -Dsbt.ivy.home=$WORKSPACE/.ivy2 \ - -Dsbt.override.build.repos=true -Dsbt.repository.config="$repositoriesFile" \ - -jar $sbtLauncher \ - $sbtDistVersionOverride \ - clean update $sbtDistTarget diff --git a/scripts/jobs/release/package/unix b/scripts/jobs/release/package/unix deleted file mode 100755 index f0f0ddc20..000000000 --- a/scripts/jobs/release/package/unix +++ /dev/null @@ -1,6 +0,0 @@ -# @pre current directory == repo root - -# to be extra-sure we don't pollute -rm -rf $WORKSPACE/.ivy2 - -. scripts/jobs/release/package/generic \ No newline at end of file diff --git a/scripts/jobs/release/package/windows b/scripts/jobs/release/package/windows deleted file mode 100755 index 3a0ec4032..000000000 --- a/scripts/jobs/release/package/windows +++ /dev/null @@ -1,6 +0,0 @@ -# @pre current directory == repo root - -# to be extra-sure we don't pollute -rm -rf "$(/usr/bin/cygpath --unix $WORKSPACE/.ivy2)" - -. scripts/jobs/release/package/generic \ No newline at end of file