diff --git a/lib/elixir_make/artefact.ex b/lib/elixir_make/artefact.ex index 787a82a..fd01180 100644 --- a/lib/elixir_make/artefact.ex +++ b/lib/elixir_make/artefact.ex @@ -286,141 +286,8 @@ defmodule ElixirMake.Artefact do end end - ## Download - - def download(url) do - url_charlist = String.to_charlist(url) - - # TODO: Remove me when we require Elixir v1.15 - {:ok, _} = Application.ensure_all_started(:inets) - {:ok, _} = Application.ensure_all_started(:ssl) - {:ok, _} = Application.ensure_all_started(:public_key) - - if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do - Mix.shell().info("Using HTTP_PROXY: #{proxy}") - %{host: host, port: port} = URI.parse(proxy) - - :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}]) - end - - if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do - Mix.shell().info("Using HTTPS_PROXY: #{proxy}") - %{host: host, port: port} = URI.parse(proxy) - :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}]) - end - - # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets - # TODO: This may no longer be necessary from Erlang/OTP 25.0 or later. - https_options = [ - ssl: - [ - verify: :verify_peer, - customize_hostname_check: [ - match_fun: :public_key.pkix_verify_hostname_match_fun(:https) - ] - ] ++ cacerts_options() - ] - - options = [body_format: :binary] - - case :httpc.request(:get, {url_charlist, []}, https_options, options) do - {:ok, {{_, 200, _}, _headers, body}} -> - {:ok, body} - - other -> - {:error, "couldn't fetch NIF from #{url}: #{inspect(other)}"} - end - end - - defp cacerts_options do - cond do - path = System.get_env("ELIXIR_MAKE_CACERT") -> - [cacertfile: path] - - certs = otp_cacerts() -> - [cacerts: certs] - - Application.spec(:castore, :vsn) -> - [cacertfile: Application.app_dir(:castore, "priv/cacerts.pem")] - - Application.spec(:certifi, :vsn) -> - [cacertfile: Application.app_dir(:certifi, "priv/cacerts.pem")] - - path = cacerts_from_os() -> - [cacertfile: path] - - true -> - warn_no_cacerts() - [] - end - end - - defp otp_cacerts do - if System.otp_release() >= "25" do - # cacerts_get/0 raises if no certs found - try do - :public_key.cacerts_get() - rescue - _ -> - nil - end - end - end - - # https_opts and related code are taken from - # https://github.com/elixir-cldr/cldr_utils/blob/v2.19.1/lib/cldr/http/http.ex - @certificate_locations [ - # Debian/Ubuntu/Gentoo etc. - "/etc/ssl/certs/ca-certificates.crt", - - # Fedora/RHEL 6 - "/etc/pki/tls/certs/ca-bundle.crt", - - # OpenSUSE - "/etc/ssl/ca-bundle.pem", - - # OpenELEC - "/etc/pki/tls/cacert.pem", - - # CentOS/RHEL 7 - "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", - - # Open SSL on MacOS - "/usr/local/etc/openssl/cert.pem", - - # MacOS & Alpine Linux - "/etc/ssl/cert.pem" - ] - - defp cacerts_from_os do - Enum.find(@certificate_locations, &File.exists?/1) - end - - defp warn_no_cacerts do - Mix.shell().error(""" - No certificate trust store was found. - - Tried looking for: #{inspect(@certificate_locations)} - - A certificate trust store is required in - order to download locales for your configuration. - Since elixir_make could not detect a system - installed certificate trust store one of the - following actions may be taken: - - 1. Install the hex package `castore`. It will - be automatically detected after recompilation. - - 2. Install the hex package `certifi`. It will - be automatically detected after recompilation. - - 3. Specify the location of a certificate trust store - by configuring it in environment variable: - - export ELIXIR_MAKE_CACERT="/path/to/cacerts.pem" - - 4. Use OTP 25+ on an OS that has built-in certificate - trust store. - """) + def download(config, url) do + downloader = config[:make_precompiler_downloader] || ElixirMake.Downloader.Httpc + downloader.download(url) end end diff --git a/lib/elixir_make/downloader.ex b/lib/elixir_make/downloader.ex new file mode 100644 index 0000000..e0aa554 --- /dev/null +++ b/lib/elixir_make/downloader.ex @@ -0,0 +1,10 @@ +defmodule ElixirMake.Downloader do + @moduledoc """ + The behaviour for downloader modules. + """ + + @doc """ + This callback should download the artefact from the given URL. + """ + @callback download(url :: String.t()) :: {:ok, iolist() | binary()} | {:error, String.t()} +end diff --git a/lib/elixir_make/downloader/httpc.ex b/lib/elixir_make/downloader/httpc.ex new file mode 100644 index 0000000..5798ce7 --- /dev/null +++ b/lib/elixir_make/downloader/httpc.ex @@ -0,0 +1,142 @@ +defmodule ElixirMake.Downloader.Httpc do + @moduledoc false + + @behaviour ElixirMake.Downloader + + @impl ElixirMake.Downloader + def download(url) do + url_charlist = String.to_charlist(url) + + # TODO: Remove me when we require Elixir v1.15 + {:ok, _} = Application.ensure_all_started(:inets) + {:ok, _} = Application.ensure_all_started(:ssl) + {:ok, _} = Application.ensure_all_started(:public_key) + + if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do + Mix.shell().info("Using HTTP_PROXY: #{proxy}") + %{host: host, port: port} = URI.parse(proxy) + + :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}]) + end + + if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do + Mix.shell().info("Using HTTPS_PROXY: #{proxy}") + %{host: host, port: port} = URI.parse(proxy) + :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}]) + end + + # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets + # TODO: This may no longer be necessary from Erlang/OTP 25.0 or later. + https_options = [ + ssl: + [ + verify: :verify_peer, + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ] + ] ++ cacerts_options() + ] + + options = [body_format: :binary] + + case :httpc.request(:get, {url_charlist, []}, https_options, options) do + {:ok, {{_, 200, _}, _headers, body}} -> + {:ok, body} + + other -> + {:error, "couldn't fetch NIF from #{url}: #{inspect(other)}"} + end + end + + defp cacerts_options do + cond do + path = System.get_env("ELIXIR_MAKE_CACERT") -> + [cacertfile: path] + + certs = otp_cacerts() -> + [cacerts: certs] + + Application.spec(:castore, :vsn) -> + [cacertfile: Application.app_dir(:castore, "priv/cacerts.pem")] + + Application.spec(:certifi, :vsn) -> + [cacertfile: Application.app_dir(:certifi, "priv/cacerts.pem")] + + path = cacerts_from_os() -> + [cacertfile: path] + + true -> + warn_no_cacerts() + [] + end + end + + defp otp_cacerts do + if System.otp_release() >= "25" do + # cacerts_get/0 raises if no certs found + try do + :public_key.cacerts_get() + rescue + _ -> + nil + end + end + end + + # https_opts and related code are taken from + # https://github.com/elixir-cldr/cldr_utils/blob/v2.19.1/lib/cldr/http/http.ex + @certificate_locations [ + # Debian/Ubuntu/Gentoo etc. + "/etc/ssl/certs/ca-certificates.crt", + + # Fedora/RHEL 6 + "/etc/pki/tls/certs/ca-bundle.crt", + + # OpenSUSE + "/etc/ssl/ca-bundle.pem", + + # OpenELEC + "/etc/pki/tls/cacert.pem", + + # CentOS/RHEL 7 + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + + # Open SSL on MacOS + "/usr/local/etc/openssl/cert.pem", + + # MacOS & Alpine Linux + "/etc/ssl/cert.pem" + ] + + defp cacerts_from_os do + Enum.find(@certificate_locations, &File.exists?/1) + end + + defp warn_no_cacerts do + Mix.shell().error(""" + No certificate trust store was found. + + Tried looking for: #{inspect(@certificate_locations)} + + A certificate trust store is required in + order to download locales for your configuration. + Since elixir_make could not detect a system + installed certificate trust store one of the + following actions may be taken: + + 1. Install the hex package `castore`. It will + be automatically detected after recompilation. + + 2. Install the hex package `certifi`. It will + be automatically detected after recompilation. + + 3. Specify the location of a certificate trust store + by configuring it in environment variable: + + export ELIXIR_MAKE_CACERT="/path/to/cacerts.pem" + + 4. Use OTP 25+ on an OS that has built-in certificate + trust store. + """) + end +end diff --git a/lib/mix/tasks/compile.elixir_make.ex b/lib/mix/tasks/compile.elixir_make.ex index 7b49c6e..6f45fc5 100644 --- a/lib/mix/tasks/compile.elixir_make.ex +++ b/lib/mix/tasks/compile.elixir_make.ex @@ -70,6 +70,11 @@ defmodule Mix.Tasks.Compile.ElixirMake do * `:make_precompiler_filename` - the filename of the compiled artefact without its extension. Defaults to the app name. + * `:make_precompiler_downloader` - a module implementing the `ElixirMake.Downloader` + behaviour. You can use this to customize how the precompiled artefacts + are downloaded, for example, to add HTTP authentication or to download + from an SFTP server. The default implementation uses `:httpc`. + * `:make_force_build` - if build should be forced even if precompiled artefacts are available. Defaults to true if the app has a `-dev` version flag. @@ -219,7 +224,7 @@ defmodule Mix.Tasks.Compile.ElixirMake do unless File.exists?(archived_fullpath) do Mix.shell().info("Downloading precompiled NIF to #{archived_fullpath}") - with {:ok, archived_data} <- Artefact.download(url) do + with {:ok, archived_data} <- Artefact.download(config, url) do File.mkdir_p(Path.dirname(archived_fullpath)) File.write(archived_fullpath, archived_data) end diff --git a/lib/mix/tasks/elixir_make.checksum.ex b/lib/mix/tasks/elixir_make.checksum.ex index 31a0648..9280d8c 100644 --- a/lib/mix/tasks/elixir_make.checksum.ex +++ b/lib/mix/tasks/elixir_make.checksum.ex @@ -84,7 +84,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do Mix.raise("you need to specify either \"--all\" or \"--only-local\" flags") end - artefacts = download_and_checksum_all(urls, options) + artefacts = download_and_checksum_all(config, urls, options) if Keyword.get(options, :print, false) do artefacts @@ -97,7 +97,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do Artefact.write_checksums!(artefacts) end - defp download_and_checksum_all(urls, options) do + defp download_and_checksum_all(config, urls, options) do ignore_unavailable? = Keyword.get(options, :ignore_unavailable, false) tasks = @@ -106,7 +106,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do fn {{_target, _nif_version}, url} -> checksum_algo = Artefact.checksum_algo() checksum_file_url = "#{url}.#{Atom.to_string(checksum_algo)}" - artifact_checksum = Artefact.download(checksum_file_url) + artifact_checksum = Artefact.download(config, checksum_file_url) with {:ok, body} <- artifact_checksum, [checksum, basename] <- String.split(body, " ", trim: true) do @@ -117,7 +117,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do checksum_algo: checksum_algo }} else - _ -> {:download, url, Artefact.download(url)} + _ -> {:download, url, Artefact.download(config, url)} end end, timeout: :infinity, diff --git a/test/fixtures/my_app/mix.exs b/test/fixtures/my_app/mix.exs index 61689a3..7493650 100644 --- a/test/fixtures/my_app/mix.exs +++ b/test/fixtures/my_app/mix.exs @@ -10,7 +10,7 @@ defmodule MyApp.Precompiler do @behaviour ElixirMake.Precompiler @impl true - def current_target, do: "target" + def current_target, do: {:ok, "target"} @impl true def all_supported_targets(_), do: ["target"] diff --git a/test/mix/tasks/compile.make_test.exs b/test/mix/tasks/compile.make_test.exs index e0cb9d9..7bed9be 100644 --- a/test/mix/tasks/compile.make_test.exs +++ b/test/mix/tasks/compile.make_test.exs @@ -13,6 +13,7 @@ defmodule Mix.Tasks.Compile.ElixirMakeTest do end setup do + Mix.Task.reenable("elixir_make.precompile") System.delete_env("MAKE") System.delete_env("ERL_EI_LIBDIR") System.delete_env("ERL_EI_INCLUDE_DIR") @@ -24,6 +25,8 @@ defmodule Mix.Tasks.Compile.ElixirMakeTest do File.rm_rf!("Makefile") File.rm_rf!("_build") File.rm_rf!("priv") + File.rm_rf!("cache") + File.rm_rf!("dl") end) :ok @@ -378,6 +381,46 @@ defmodule Mix.Tasks.Compile.ElixirMakeTest do end) end + test "custom downloader" do + in_fixture(fn -> + File.mkdir!("priv") + + File.write("Makefile", """ + all: + \t@touch priv/my_app + \t@echo "all" + """) + + with_project_config( + [ + make_precompiler: {:nif, MyApp.Precompiler}, + make_precompiler_url: "https://example.com/@{artefact_filename}", + make_precompiler_downloader: MyApp.Downloader + ], + fn -> + custom_downloader_dir = "./dl" + cache_dir = "./cache" + System.put_env("ELIXIR_MAKE_CACHE_DIR", cache_dir) + + # precompile + output = + capture_io(fn -> + Mix.Tasks.ElixirMake.Precompile.run([]) + end) + + assert output =~ "all\n" + + # move cache to custom downloader dir + File.rename!(cache_dir, custom_downloader_dir) + + # artifacts should be "downloaded" by the custom downloader + output = capture_io(fn -> run([]) end) + assert output =~ "Custom downloader downloading from https://example.com/my_app-nif-" + end + ) + end) + end + defp in_fixture(fun) do File.cd!(@fixture_project, fun) end @@ -386,3 +429,14 @@ defmodule Mix.Tasks.Compile.ElixirMakeTest do Mix.Project.in_project(:my_app, @fixture_project, config, fn _ -> fun.() end) end end + +defmodule MyApp.Downloader do + @behaviour ElixirMake.Downloader + + @impl true + def download(url) do + IO.puts("Custom downloader downloading from #{url}") + path = String.replace(url, "https://example.com/", __DIR__ <> "/dl/") + File.read(path) + end +end