From 9756970ae4810bb029326f146e5800ec2dc76ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Dro=C5=BCd=C5=BCy=C5=84ski?= Date: Thu, 24 Sep 2026 14:32:09 +0200 Subject: [PATCH 1/5] Support Elixir 1.16+ and drop erlexec and tesla dependencies - Require Elixir ~> 1.16 and bump version to 0.5.0 - Update all dependencies, drop coverex and mix_test_watch - Remove Bow.Exec and erlexec; add Bow.with_output/2 helper so versions can be generated with any tool (System.cmd, Vix, MuonTrap, ...) - Remove tesla; add Bow.Downloader behaviour with default :httpc based implementation streaming downloads to disk - Bow.Download no longer depends on Ecto - Use Config instead of Mix.Config, configurable test DB host/port - Add GitHub Actions CI with Postgres and S3 mock services --- .github/workflows/ci.yml | 76 +++++++++++++++ .tool-versions | 2 + CHANGELOG.md | 23 +++++ README.md | 77 +++++++++++++--- config/config.exs | 10 +- lib/bow.ex | 64 ++++++++++++- lib/bow/download.ex | 115 +++++++++++++---------- lib/bow/downloader.ex | 49 ++++++++++ lib/bow/downloader/httpc.ex | 108 ++++++++++++++++++++++ lib/bow/ecto.ex | 30 +++--- lib/bow/exec.ex | 101 -------------------- lib/bow/uploader.ex | 25 +++-- mix.exs | 45 +++------ mix.lock | 69 ++++++-------- test/bow/download_test.exs | 142 +++++++++-------------------- test/bow/downloader/httpc_test.exs | 111 ++++++++++++++++++++++ test/bow/ecto_test.exs | 35 ++----- test/bow/exec_test.exs | 52 ----------- test/bow/with_output_test.exs | 80 ++++++++++++++++ test/bow_test.exs | 2 +- test/scripts/copy.sh | 3 - test/scripts/fail.sh | 2 - test/scripts/sleep.sh | 2 - test/support/test_downloader.ex | 33 +++++++ 24 files changed, 803 insertions(+), 453 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .tool-versions create mode 100644 lib/bow/downloader.ex create mode 100644 lib/bow/downloader/httpc.ex delete mode 100644 lib/bow/exec.ex create mode 100644 test/bow/downloader/httpc_test.exs delete mode 100644 test/bow/exec_test.exs create mode 100644 test/bow/with_output_test.exs delete mode 100755 test/scripts/copy.sh delete mode 100755 test/scripts/fail.sh delete mode 100755 test/scripts/sleep.sh create mode 100644 test/support/test_downloader.ex diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5915d96 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +jobs: + test: + name: Elixir ${{ matrix.elixir }} / OTP ${{ matrix.otp }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + # minimum supported + - elixir: "1.16" + otp: "26" + # latest + - elixir: "1.20" + otp: "28" + lint: true + env: + MIX_ENV: test + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: development + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + s3: + image: adobe/s3mock + env: + COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: test-bucket + ports: + - 4567:9090 + steps: + - uses: actions/checkout@v5 + + - uses: erlef/setup-beam@v1 + with: + elixir-version: ${{ matrix.elixir }} + otp-version: ${{ matrix.otp }} + + - uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ matrix.elixir }}-${{ matrix.otp }}-${{ hashFiles('mix.lock') }} + restore-keys: ${{ runner.os }}-mix-${{ matrix.elixir }}-${{ matrix.otp }}- + + - run: mix deps.get + + - if: matrix.lint + run: mix format --check-formatted + + - if: matrix.lint + run: mix deps.unlock --check-unused + + - if: matrix.lint + run: mix compile --warnings-as-errors + + - run: mix ecto.create + + - name: Wait for S3 mock + run: timeout 60 bash -c 'until curl -sf http://localhost:4567/ > /dev/null; do sleep 1; done' + + - run: mix test --include ecto --include s3 diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..a7baa92 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +erlang 28.5.0.2 +elixir 1.19.5-otp-28 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc93b6..fa02550 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +# v0.5.0 + +Breaking changes: + +* Minimum Elixir version is now 1.16 +* `Bow.Exec` and `erlexec` dependency removed. Generate versions in `transform/3` with + `Bow.with_output/2` and any tool you like (`System.cmd/3`, Vix, MuonTrap, erlexec...). + `:exec_timeout` config removed +* `tesla` dependency removed. Remote files are downloaded with `:httpc` by default (`Bow.Downloader.Httpc`), + use `config :bow, downloader: MyDownloader` to plug in your own `Bow.Downloader` (e.g. Req or Tesla) +* `Bow.Download.download(client, url)` is now `Bow.Download.download(url, opts)` +* `Bow.Ecto.cast_uploads/4` and `Bow.Ecto.download_params/3` take options (e.g. `downloader: ...`) + instead of a Tesla client +* `Bow.Download.download/2` errors are `{:error, %{status: status, headers: headers}}` + and `{:error, :too_many_redirects}` + +Other changes: + +* Updated dependencies (Ecto 3.10+, Plug 1.14+, ExAws 2.4+) +* Add `Bow.with_output/2` helper for writing processed files +* Downloaded files are streamed to disk instead of being loaded into memory +* fix: `Bow.Download` no longer requires Ecto + # v0.4.3 * fix: handle valid URL without file path diff --git a/README.md b/README.md index edd33ff..b3d78c9 100644 --- a/README.md +++ b/README.md @@ -14,18 +14,17 @@ File uploads for Elixir ```elixir def deps do [ - {:bow, "~> 0.4.3"}, + {:bow, "~> 0.5.0"}, # for AWS S3 support - {:ex_aws, "~> 2.0"}, - {:ex_aws_s3, "~> 2.0"}, - - # for Bow.Exec - {:erlexec, "~> 1.7.0"} + {:ex_aws, "~> 2.4"}, + {:ex_aws_s3, "~> 2.4"} ] end ``` +Bow requires Elixir 1.16 or newer. + ## Usage ### Minimal uploader definition @@ -65,8 +64,12 @@ defmodule AttachmentUploader do # generate image thumbnail def transform(source, target, :thumb) do - Bow.Exec.exec source, target, - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) end @@ -186,6 +189,54 @@ defmodule AvatarUploader do end ``` +### Downloading remote files + +`Bow.Ecto.cast_uploads/4` downloads files given as `remote__url` params +(e.g. `remote_avatar_url`). Files are downloaded with Erlang `:httpc` by default, +you can use any HTTP client by implementing the `Bow.Downloader` behaviour: + +```elixir +# config/config.exs +config :bow, downloader: MyApp.BowDownloader +``` + +See `Bow.Downloader` docs for an example based on Req. + +### Processing files + +`transform/3` can generate the version in any way you like. `Bow.with_output/2` helps with +the boring part: it gives you a temporary output path (with the target extension, so tools like +ImageMagick or libvips pick the right format), checks the file was written and sets it on the target. + +With [Vix](https://hexdocs.pm/vix): + +```elixir +def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + with {:ok, thumb} <- Vix.Vips.Operation.thumbnail(source.path, 250) do + Vix.Vips.Image.write_to_file(thumb, output_path) + end + end) +end +``` + +With an external program: + +```elixir +def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) +end +``` + +`System.cmd/3` does not support timeouts. Version generation is limited by `:version_timeout`, +but the OS process may keep running after it. If you need to kill it, use a library like +[MuonTrap](https://hexdocs.pm/muontrap) or [erlexec](https://hexdocs.pm/erlexec). + ### Using Bow in test environment It is best to use local storage adapter when testing. @@ -206,7 +257,8 @@ mix test #### Testing ecto integration ```bash -# edit config/config.exs +# start postgres, or use your own and set TEST_DB_USERNAME, TEST_DB_PASSWORD, TEST_DB_HOST, TEST_DB_PORT +docker run --rm -d -p 5432:5432 -e POSTGRES_USER=development -e POSTGRES_HOST_AUTH_METHOD=trust postgres:16-alpine # create test database MIX_ENV=test mix ecto.create @@ -218,11 +270,8 @@ mix test --only ecto #### Testing S3 adapter ```bash -# install fake-s3 gem -gem install fakes3 - -# start fake-s3 server -fakes3 -r tmp/s3 -p 4567 +# start S3 mock server +docker run --rm -d -p 4567:9090 -e COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS=test-bucket adobe/s3mock # run tests mix test --only s3 diff --git a/config/config.exs b/config/config.exs index 4e02310..71e0cb4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,22 +1,24 @@ -use Mix.Config +import Config if Mix.env() == :test do config :bow, Bow.Repo, username: System.get_env("TEST_DB_USERNAME") || "development", password: System.get_env("TEST_DB_PASSWORD") || "", database: "bow_ecto_test", - hostname: "localhost", + hostname: System.get_env("TEST_DB_HOST") || "localhost", + port: String.to_integer(System.get_env("TEST_DB_PORT") || "5432"), pool: Ecto.Adapters.SQL.Sandbox config :bow, ecto_repos: [Bow.Repo] - config :logger, level: :warn + config :logger, level: :warning # fakes3 configuration # https://github.com/jubos/fake-s3/wiki/Supported-Clients#elixir config :ex_aws, access_key_id: ["123", :instance_role], secret_access_key: ["asdf", :instance_role], - region: "fakes3" + region: "fakes3", + http_client: ExAws.Request.Req config :ex_aws, :s3, scheme: "http://", diff --git a/lib/bow.ex b/lib/bow.ex index f9dc433..8473096 100644 --- a/lib/bow.ex +++ b/lib/bow.ex @@ -9,7 +9,7 @@ defmodule Bow do storage_prefix: "priv/static/uploads", # storage directory prefix store_timeout: 30_000, # single version upload timeout - exec_timeout: 15_000, # single command execution timeout + version_timeout: 60_000, # single version processing timeout """ def storage, do: Application.get_env(:bow, :storage, Bow.Storage.Local) @@ -108,6 +108,68 @@ defmodule Bow do end end + @doc """ + Run `fun` with an output path for `target` and set the result on it + + Use it in `c:Bow.Uploader.transform/3` to write the processed file. + The output path has the same extension as `target` name, so tools choosing + the format based on extension (like libvips or ImageMagick) work out of the box. + The file is removed when the calling process exits. + + `fun` must write the file and return `:ok` or `{:error, reason}`. + + # thumbnail with Vix + def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + with {:ok, thumb} <- Vix.Vips.Operation.thumbnail(source.path, 250) do + Vix.Vips.Image.write_to_file(thumb, output_path) + end + end) + end + + # thumbnail with ImageMagick + def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path], stderr_to_stdout: true) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) + end + """ + @spec with_output(t, (output_path :: Path.t() -> :ok | {:error, any})) :: + {:ok, t} | {:error, any} + def with_output(target, fun) when is_function(fun, 1) do + tmp_path = Plug.Upload.random_file!("bow-output") + # random_file! does not support extensions, write output next to it and move it back + # so the file is still removed together with the process + output_path = tmp_path <> target.ext + + case fun.(output_path) do + :ok -> + keep_output(target, output_path, tmp_path) + + {:error, reason} -> + File.rm(output_path) + {:error, reason} + + other -> + File.rm(output_path) + + raise ArgumentError, + "expected :ok or {:error, reason} from with_output/2 function, got: #{inspect(other)}" + end + end + + defp keep_output(target, output_path, tmp_path) do + if File.exists?(output_path) do + File.rename!(output_path, tmp_path) + {:ok, set(target, :path, tmp_path)} + else + {:error, :output_not_found} + end + end + defp make(up, f0, fx, versions, opts) when is_list(versions) do versions |> Enum.map(&Task.async(fn -> make(up, f0, fx, &1, opts) end)) diff --git a/lib/bow/download.ex b/lib/bow/download.ex index 168e9ad..dd59954 100644 --- a/lib/bow/download.ex +++ b/lib/bow/download.ex @@ -1,52 +1,74 @@ defmodule Bow.Download do - use Tesla, only: [:get] - - plug(Tesla.Middleware.FollowRedirects) + @max_redirects 5 + @redirect_statuses [301, 302, 303, 307, 308] @doc """ Download file from given URL + + The file name is taken from the URL (after following redirects) + with extension based on the `content-type` header. + + Options: + - `:downloader` - module implementing `Bow.Downloader`, defaults to `:downloader` config + or `Bow.Downloader.Httpc` + - `:max_redirects` - maximum number of redirects to follow, defaults to `#{@max_redirects}` + + Other options are passed to the downloader. """ - @spec download(client :: Tesla.Client.t(), url :: String.t()) :: - {:ok, Bow.t()} | {:error, any} - def download(client \\ %Tesla.Client{}, url) do - case get!(client, encode(url)) do - %{status: 200, url: url, body: body} = env -> - base = - url - |> URI.parse() - |> case do - %{path: path} when not is_nil(path) -> path |> Path.basename() - _ -> "" - end - - name = - case Tesla.get_header(env, "content-type") do - nil -> - base - - content_type -> - case MIME.extensions(content_type) do - [ext | _] -> rootname(base) <> "." <> ext - _ -> base - end - end - - path = Plug.Upload.random_file!("bow-download") - - case File.write(path, body) do - :ok -> - {:ok, Bow.new(name: name, path: path)} - - {:error, reason} -> - {:error, reason} - end - - env -> - {:error, env} + @spec download(url :: String.t(), opts :: keyword) :: {:ok, Bow.t()} | {:error, any} + def download(url, opts \\ []) do + {downloader, opts} = Keyword.pop(opts, :downloader, downloader()) + {max_redirects, opts} = Keyword.pop(opts, :max_redirects, @max_redirects) + path = Plug.Upload.random_file!("bow-download") + + with {:ok, url, headers} <- get(downloader, encode(url), path, opts, max_redirects) do + {:ok, Bow.new(name: name(url, headers), path: path)} + end + end + + defp downloader, do: Application.get_env(:bow, :downloader, Bow.Downloader.Httpc) + + defp get(downloader, url, path, opts, redirects_left) do + case downloader.get(url, path, opts) do + {:ok, 200, headers} -> + {:ok, url, headers} + + {:ok, status, _headers} when status in @redirect_statuses and redirects_left == 0 -> + {:error, :too_many_redirects} + + {:ok, status, headers} when status in @redirect_statuses -> + follow_redirect(downloader, url, path, opts, redirects_left, status, headers) + + {:ok, status, headers} -> + {:error, %{status: status, headers: headers}} + + {:error, reason} -> + {:error, reason} + end + end + + defp follow_redirect(downloader, url, path, opts, redirects_left, status, headers) do + with {_, location} <- List.keyfind(headers, "location", 0) do + next_url = url |> URI.merge(location) |> to_string() + get(downloader, next_url, path, opts, redirects_left - 1) + else + nil -> {:error, %{status: status, headers: headers}} + end + end + + defp name(url, headers) do + base = + case URI.parse(url) do + %{path: path} when not is_nil(path) -> Path.basename(path) + _ -> "" + end + + with {_, content_type} <- List.keyfind(headers, "content-type", 0), + [ext | _] <- MIME.extensions(content_type) do + rootname(base) <> "." <> ext + else + _ -> base end - rescue - ex in Tesla.Error -> - {:error, ex} end # If path name is malformed, for example looks like this: ".some-data", @@ -57,11 +79,8 @@ defmodule Bow.Download do # name if we cannot read it properly. defp rootname(base) do case Path.rootname(base) do - "" -> - Ecto.UUID.generate() - - name -> - name + "" -> 16 |> :crypto.strong_rand_bytes() |> Base.encode16(case: :lower) + name -> name end end diff --git a/lib/bow/downloader.ex b/lib/bow/downloader.ex new file mode 100644 index 0000000..77b359c --- /dev/null +++ b/lib/bow/downloader.ex @@ -0,0 +1,49 @@ +defmodule Bow.Downloader do + @moduledoc """ + Behaviour for HTTP clients used by `Bow.Download` + + Bow uses `Bow.Downloader.Httpc` (based on Erlang `:httpc`) by default. + You can provide your own implementation, e.g. to use a different HTTP client, + a proxy or to mock requests in tests: + + config :bow, downloader: MyApp.BowDownloader + + The downloader can also be passed per call with the `:downloader` option + of `Bow.Download.download/2` and `Bow.Ecto.cast_uploads/4`. + + ## Example + + Implementation based on [Req](https://hexdocs.pm/req): + + defmodule MyApp.BowDownloader do + @behaviour Bow.Downloader + + @impl true + def get(url, path, _opts) do + case Req.get(url, redirect: false, into: File.stream!(path)) do + {:ok, %{status: status, headers: headers}} -> + {:ok, status, for({name, values} <- headers, value <- values, do: {name, value})} + + {:error, reason} -> + {:error, reason} + end + end + end + """ + + @type headers :: [{name :: String.t(), value :: String.t()}] + + @doc """ + Make a single GET request + + Arguments: + - `url` - URL to request + - `path` - path the response body must be written to when the status is 2xx + - `opts` - options given to `Bow.Download.download/2` + + Must not follow redirects, `Bow.Download` takes care of them. + Must return the response status and headers with lowercase names. + """ + @callback get(url :: String.t(), path :: Path.t(), opts :: keyword) :: + {:ok, status :: pos_integer, headers} | {:error, any} +end diff --git a/lib/bow/downloader/httpc.ex b/lib/bow/downloader/httpc.ex new file mode 100644 index 0000000..c87c83f --- /dev/null +++ b/lib/bow/downloader/httpc.ex @@ -0,0 +1,108 @@ +defmodule Bow.Downloader.Httpc do + @moduledoc """ + Default `Bow.Downloader` based on Erlang `:httpc` + + The response body is streamed directly to the file. + HTTPS certificates are verified using the operating system CA store. + + Options: + - `:timeout` - request timeout in milliseconds, defaults to `30_000` + """ + + @behaviour Bow.Downloader + + @default_timeout 30_000 + + @impl true + def get(url, path, opts) do + timeout = Keyword.get(opts, :timeout, @default_timeout) + request = {String.to_charlist(url), [{~c"user-agent", ~c"bow"}]} + + http_opts = [ + timeout: timeout, + connect_timeout: timeout, + autoredirect: false, + ssl: ssl_opts() + ] + + case :httpc.request(:get, request, http_opts, + sync: false, + stream: :self, + body_format: :binary + ) do + {:ok, ref} -> receive_response(ref, path, timeout) + {:error, reason} -> {:error, reason} + end + end + + # :httpc streams only 200 and 206 responses, others are received at once + defp receive_response(ref, path, timeout) do + receive do + {:http, {^ref, :stream_start, headers}} -> + save_stream(ref, path, headers, timeout) + + {:http, {^ref, {{_version, status, _reason}, headers, body}}} -> + save_body(path, status, headers, body) + + {:http, {^ref, {:error, reason}}} -> + {:error, reason} + after + timeout -> cancel(ref, :timeout) + end + end + + defp save_stream(ref, path, headers, timeout) do + with {:ok, :ok} <- File.open(path, [:write, :binary], &stream_body(ref, &1, timeout)) do + {:ok, 200, normalize(headers)} + else + {:ok, {:error, reason}} -> {:error, reason} + {:error, reason} -> cancel(ref, reason) + end + end + + defp save_body(path, status, headers, body) when status in 200..299 do + with :ok <- File.write(path, body) do + {:ok, status, normalize(headers)} + end + end + + defp save_body(_path, status, headers, _body), do: {:ok, status, normalize(headers)} + + defp stream_body(ref, file, timeout) do + receive do + {:http, {^ref, :stream, chunk}} -> + IO.binwrite(file, chunk) + stream_body(ref, file, timeout) + + {:http, {^ref, :stream_end, _headers}} -> + :ok + + {:http, {^ref, {:error, reason}}} -> + {:error, reason} + after + timeout -> cancel(ref, :timeout) + end + end + + defp cancel(ref, reason) do + :httpc.cancel_request(ref) + {:error, reason} + end + + defp normalize(headers) do + for {name, value} <- headers do + {name |> to_string() |> String.downcase(), to_string(value)} + end + end + + defp ssl_opts do + [ + verify: :verify_peer, + cacerts: :public_key.cacerts_get(), + depth: 4, + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ] + ] + end +end diff --git a/lib/bow/ecto.ex b/lib/bow/ecto.ex index 2aeb0f5..9d7fd8d 100644 --- a/lib/bow/ecto.ex +++ b/lib/bow/ecto.ex @@ -303,34 +303,28 @@ defmodule Bow.Ecto do Download remote files for given fields, i.e. `params["remote_avatar_url"] = "http://example.com/some/file.png"` + Options are passed to `Bow.Download.download/2`, e.g. `:downloader`. + Example changeset |> cast(params, [:name, :avatar]) |> Bow.Ecto.cast_uploads(params, [:avatar]) """ - @spec cast_uploads(any, map, list, Tesla.Client.t()) :: - Ecto.Changeset.t() - def cast_uploads(changeset, params, fields, client \\ %Tesla.Client{}) do - Ecto.Changeset.cast(changeset, download_params(params, fields, client), fields) + @spec cast_uploads(any, map, list, keyword) :: Ecto.Changeset.t() + def cast_uploads(changeset, params, fields, opts \\ []) do + Ecto.Changeset.cast(changeset, download_params(params, fields, opts), fields) end - @spec download_params(map, list, Tesla.Client.t()) :: map - def download_params(params, fields, client \\ %Tesla.Client{}) do + @spec download_params(map, list, keyword) :: map + def download_params(params, fields, opts \\ []) do Enum.reduce(fields, params, fn field, params -> field = to_string(field) - case params["remote_#{field}_url"] do - nil -> - params - - "" -> - params - - url -> - case Bow.Download.download(client, url) do - {:ok, file} -> Map.put(params, field, file) - _ -> params - end + with url when url not in [nil, ""] <- params["remote_#{field}_url"], + {:ok, file} <- Bow.Download.download(url, opts) do + Map.put(params, field, file) + else + _ -> params end end) end diff --git a/lib/bow/exec.ex b/lib/bow/exec.ex deleted file mode 100644 index ce0a6c3..0000000 --- a/lib/bow/exec.ex +++ /dev/null @@ -1,101 +0,0 @@ -defmodule Bow.Exec do - @moduledoc """ - Transform files with shell commands - - This module allows executing any external command taking care of temporary path generation and error handling. - It is as reliable as [erlexec](https://github.com/saleyn/erlexec) module (very!). - - It is also possible to provide custom command timeout. See `exec/4` to see all available options. - """ - - @type command :: [String.t() | {:input, integer} | :input | :output] - - defp default_timeout, do: Application.get_env(:bow, :exec_timeout, 15_000) - - @doc """ - Execute command - - Arguments: - - `source` - source file to be transformed - - `target_name` - target file - - `command` - the command to be executed. Placeholders `${input}` and `${output}` will be replaced with source and target paths - - Options: - - `:timeout` - time in which the command must return. If it's exceeded the command process will be killed. - - Examples - - # generate image thumbnail from first page of pdf - def transform(file, :pdf_thumbnail) do - Bow.Exec.exec file, filename(file, :pdf_thumbnail), - "convert '${input}[0]' -strip -gravity North -background '#ffffff'" <> - " -resize 250x175^ -extent 250x175 -format png png:${output}" - end - - """ - @spec exec(Bow.t(), Bow.t(), command, keyword) :: {:ok, Bow.t()} | {:error, any} - def exec(source, target, command, opts \\ []) do - timeout = opts[:timeout] || default_timeout() - - source_path = source.path - target_path = Plug.Upload.random_file!("bow-exec") - - cmd = - command - |> Enum.map(fn - {:input, idx} when is_integer(idx) -> "#{source_path}[#{idx}]" - :input -> source_path - :output -> target_path - {:output, format} -> "#{format}:#{target_path}" - arg -> arg - end) - |> Enum.map(&to_charlist/1) - - trapping(fn -> - case :exec.run_link(cmd, stdout: self(), stderr: self()) do - {:ok, pid, ospid} -> - case wait_for_exit(pid, ospid, timeout) do - {:ok, output} -> - if File.exists?(target_path) do - {:ok, Bow.set(target, :path, target_path)} - else - {:error, reason: :file_not_found, output: output, exit_code: 0, cmd: cmd} - end - - {:error, exit_code, output} -> - {:error, output: output, exit_code: exit_code, cmd: cmd} - end - - error -> - error - end - end) - end - - defp trapping(fun) do - trap = Process.flag(:trap_exit, true) - result = fun.() - Process.flag(:trap_exit, trap) - result - end - - defp wait_for_exit(pid, ospid, timout) do - receive do - {:EXIT, ^pid, :normal} -> {:ok, receive_output(ospid)} - {:EXIT, ^pid, {:exit_status, code}} -> {:error, code, receive_output(ospid)} - after - timout -> - :exec.stop_and_wait(pid, 2000) - {:error, :timeout, receive_output(ospid)} - end - end - - defp receive_output(ospid, output \\ []) do - receive do - {:stdout, ^ospid, data} -> receive_output(ospid, [output, data]) - {:stderr, ^ospid, data} -> receive_output(ospid, [output, data]) - after - 0 -> output |> to_string - end - end -end diff --git a/lib/bow/uploader.ex b/lib/bow/uploader.ex index b71b881..91c72d3 100644 --- a/lib/bow/uploader.ex +++ b/lib/bow/uploader.ex @@ -40,8 +40,12 @@ defmodule Bow.Uploader do # generate image thumbnail def transform(source, target, :thumb) do - Bow.Exec.exec source, target, - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) end @@ -178,11 +182,15 @@ defmodule Bow.Uploader do defmodule MyImageUploader do # generate image thumbnail def transform(source, target, :thumb) do - # Bow.Exec allows executing any system command replacing ${input} and ${output} - # with correct paths. It can also take :timeout option to prevent resource consumtion. - # Refer to Bow.Exec documentation for more details - Bow.Exec.exec source, target, - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" + # Bow.with_output gives a temporary output path (with target extension) + # and sets it on target when the file was written. + # Refer to Bow.with_output/2 documentation for more details + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) end end @@ -191,8 +199,7 @@ defmodule Bow.Uploader do defmodule MyImageUploader do # generate image thumbnail and then micro_thumb version based on that def transform(source, target, :thumb) do - with {:ok, thumb_file} <- Bow.Exec.exec source, target - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" do + with {:ok, thumb_file} <- Bow.with_output(target, &make_thumb(source.path, &1)) do {:ok, thumb_file, [:micro_thumb]} end end diff --git a/mix.exs b/mix.exs index 740989b..67fb722 100644 --- a/mix.exs +++ b/mix.exs @@ -1,18 +1,17 @@ defmodule Bow.Mixfile do use Mix.Project - @version "0.4.3" + @version "0.5.0" def project do [ app: :bow, version: @version, - elixir: "~> 1.4", + elixir: "~> 1.16", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, aliases: aliases(), deps: deps(), - test_coverage: [tool: Coverex.Task], package: package(), dialyzer: dialyzer(), @@ -24,46 +23,30 @@ defmodule Bow.Mixfile do def application do [ - extra_applications: [:logger] ++ applications(Mix.env()) + extra_applications: [:logger, :crypto, :inets, :ssl, :public_key] ] end defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] - defp applications(:test) do - [ - # Bow.Ecto - :ecto, - :postgrex, - # Bow.Exec - :erlexec, - # Bow.Storage.S3 - :hackney, - :sweet_xml - ] - end - - defp applications(_), do: [] - defp deps do [ - {:plug, "~> 1.0"}, - {:tesla, "~> 1.0"}, - {:ecto, "~> 3.2", optional: true}, - {:ecto_sql, "~> 3.2", optional: true}, - {:erlexec, "~> 2.0", optional: true}, - {:ex_aws, "~> 2.0", optional: true}, - {:ex_aws_s3, "~> 2.0", optional: true}, + {:plug, "~> 1.14"}, + {:mime, "~> 1.0 or ~> 2.0"}, + {:ecto, "~> 3.10", optional: true}, + {:ecto_sql, "~> 3.10", optional: true}, + {:ex_aws, "~> 2.4", optional: true}, + {:ex_aws_s3, "~> 2.4", optional: true}, {:sweet_xml, "~> 0.7", optional: true}, # testing & docs {:jason, "~> 1.4", only: :test}, + {:req, "~> 0.5", only: :test}, + {:bandit, "~> 1.0", only: :test}, {:postgrex, ">= 0.0.0", only: :test}, - {:coverex, "~> 1.5", only: :test}, - {:ex_doc, "~> 0.21", only: :dev}, - {:mix_test_watch, "~> 1.1", only: :dev}, - {:dialyxir, "~> 1.1", only: :dev} + {:ex_doc, "~> 0.34", only: :dev, runtime: false}, + {:dialyxir, "~> 1.4", only: :dev, runtime: false} ] end @@ -96,7 +79,7 @@ defmodule Bow.Mixfile do defp dialyzer do [ - plt_add_apps: [:ecto, :ex_aws, :ex_aws_s3, :erlexec] + plt_add_apps: [:ecto, :ex_aws, :ex_aws_s3] ] end end diff --git a/mix.lock b/mix.lock index 5602677..1b2fa48 100644 --- a/mix.lock +++ b/mix.lock @@ -1,41 +1,32 @@ %{ - "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"}, - "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "coverex": {:hex, :coverex, "1.5.0", "a4248302f09562993041f1b056866bfd5688d3a03c429de80c47ea6663989ecc", [:mix], [{:hackney, "~> 1.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 3.1 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "21a8f6e734a277b86c01b3cc44b7cc8b77cb1886adbebb708eefc6398567dcac"}, - "db_connection": {:hex, :db_connection, "2.4.1", "6411f6e23f1a8b68a82fa3a36366d4881f21f47fc79a9efb8c615e62050219da", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ea36d226ec5999781a9a8ad64e5d8c4454ecedc7a4d643e4832bf08efca01f00"}, - "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, - "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, - "earmark": {:hex, :earmark, "1.4.4", "4821b8d05cda507189d51f2caeef370cf1e18ca5d7dfb7d31e9cafe6688106a4", [:mix], [], "hexpm", "1f93aba7340574847c0f609da787f0d79efcab51b044bb6e242cae5aca9d264d"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.18", "e1b2be73eb08a49fb032a0208bf647380682374a725dfb5b9e510def8397f6f2", [:mix], [], "hexpm", "114a0e85ec3cf9e04b811009e73c206394ffecfcc313e0b346de0d557774ee97"}, - "ecto": {:hex, :ecto, "3.7.1", "a20598862351b29f80f285b21ec5297da1181c0442687f9b8329f0445d228892", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d36e5b39fc479e654cffd4dbe1865d9716e4a9b6311faff799b6f90ab81b8638"}, - "ecto_sql": {:hex, :ecto_sql, "3.7.1", "8de624ef50b2a8540252d8c60506379fbbc2707be1606853df371cf53df5d053", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.7.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2b42a32e2ce92f64aba5c88617891ab3b0ba34f3f3a503fa20009eae1a401c81"}, - "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "erlexec": {:hex, :erlexec, "2.2.2", "edb9f1a7d821a9df4efdc82bd11817ab5249dbd294d5c54ec249bd905f10e804", [:rebar3], [], "hexpm", "5e8e3c3773113785361b3b55218d92f7e91509cc9d679bf67c5c3703b394c900"}, - "ex_aws": {:hex, :ex_aws, "2.2.8", "8c142e516ab7c40e4eac1e2dbcda4c42ced48ff58ced81863fe4b7317c4de92b", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "84b51866078cc06970e608e4f57b4af246bb521ae650c284e82a7568bf436354"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.3.2", "92a63b72d763b488510626d528775b26831f5c82b066a63a3128054b7a09de28", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "b235b27131409bcc293c343bf39f1fbdd32892aa237b3f13752e914dc2979960"}, - "ex_doc": {:hex, :ex_doc, "0.26.0", "1922164bac0b18b02f84d6f69cab1b93bc3e870e2ad18d5dacb50a9e06b542a3", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2775d66e494a9a48355db7867478ffd997864c61c65a47d31c4949459281c78d"}, - "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm", "9a00246e8af58cdf465ae7c48fd6fd7ba2e43300413dfcc25447ecd3bf76f0c1"}, - "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mix_test_watch": {:hex, :mix_test_watch, "1.1.0", "330bb91c8ed271fe408c42d07e0773340a7938d8a0d281d57a14243eae9dc8c3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "52b6b1c476cbb70fd899ca5394506482f12e5f6b0d6acff9df95c7f1e0812ec3"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, - "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, - "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"}, - "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, - "poison": {:hex, :poison, "4.0.1", "bcb755a16fac91cad79bfe9fc3585bb07b9331e50cfe3420a24bcc2d735709ae", [:mix], [], "hexpm", "ba8836feea4b394bb718a161fc59a288fe0109b5006d6bdf97b6badfcf6f0f25"}, - "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.15.13", "7794e697481799aee8982688c261901de493eb64451feee6ea58207d7266d54a", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "3ffb76e1a97cfefe5c6a95632a27ffb67f28871c9741fb585f9d1c3cd2af70f1"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, - "sweet_xml": {:hex, :sweet_xml, "0.7.2", "4729f997286811fabdd8288f8474e0840a76573051062f066c4b597e76f14f9f", [:mix], [], "hexpm", "6894e68a120f454534d99045ea3325f7740ea71260bc315f82e29731d570a6e8"}, - "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, - "tesla": {:hex, :tesla, "1.4.3", "f5a494e08fb1abe4fd9c28abb17f3d9b62b8f6fc492860baa91efb1aab61c8a0", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "e0755bb664bf4d664af72931f320c97adbf89da4586670f4864bf259b5750386"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "bandit": {:hex, :bandit, "1.12.5", "af205a8e550f304caae09a97d29fd3c79a7f337526ea7cd772d2ff11d2f7c800", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.5", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "c5684ca062fa407cac115aec3256383f3e2ec9fdced7904d59cf5a7bb7ed6181"}, + "db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"}, + "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, + "dialyxir": {:hex, :dialyxir, "1.4.8", "7ef671a8aff9948b091d8c30f09467fbb16e77305cda451bce48109a0f5e021c", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "cbd5a851571e5dfeb32aaf2e840bfa98b7864cb3071bf2ef5d95d1276b12e072"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, + "ecto": {:hex, :ecto, "3.14.2", "99db28a864293a789c970651de711e3cae184291e0e7ea1166c54055ac41c1f3", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25d60b8c816a07d19d85b80bdf60978bd8b102209dda198d768cd7c6745339a6"}, + "ecto_sql": {:hex, :ecto_sql, "3.14.0", "06446ab8410d2f85bfbb80857ee224ab3b693700cbb38f6535d507449a627b2e", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.14.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.8", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f4d8d36faf294c9417b5a37ec7ac8217ee2abdef5fcf197ba690f361548d3949"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, + "ex_aws": {:hex, :ex_aws, "2.7.0", "e6bfd4b5fb8c791aa6c7d57fc7c45f050bb89de9576f1f6104aa974b234c01ec", [:mix], [{:configparser_ex, "~> 5.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 4.0", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bfe9d744d4fd4c1f40314ee7fab504d5547d1f01cd377fff1568cbe630b06d65"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.9", "862b7792f2e60d7010e2920d79964e3fab289bc0fd951b0ba8457a3f7f9d1199", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "a480d2bb2da64610014021629800e1e9457ca5e4a62f6775bffd963360c2bf90"}, + "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"}, + "finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"}, + "hpax": {:hex, :hpax, "1.1.0", "782931867cc23217c68fb5f68fe1a11f5e7544c7fda82c8a7019a5df5a4a1cdf", [:mix], [], "hexpm", "0b8d0f05832f55571d65ac720f79bf8994138ffbb133209dc4685eae0ad456a8"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mint": {:hex, :mint, "1.10.1", "c53e70867cf74017716884d8d33e0742b08b32e9cdb0031cbc69a429dc5555e3", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "0ba2a904605ed8406393444fb8b3356dc58eb59ee6c7fb94ac3f015e1be129e8"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "plug": {:hex, :plug, "1.20.3", "56c480c633ec2ce10140e236e15233bf576e1d323887d7c96711bd02ab5160db", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "be266aee1b8536ef6409d58cf39a3121319f0ec47cfa1b24024485aa0e76ad76"}, + "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, + "postgrex": {:hex, :postgrex, "0.22.4", "d271f595dfd25230b6398354e19d17bb5e2d20130fd2d9bdca7e15f125d43552", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "4aae45a2d60e35b04eea2602440be152fae332901f1fc7a60fc7cb7f0f9a9c5a"}, + "req": {:hex, :req, "0.7.4", "23e9ffec17de032a46a4b15ed65c09793893bf4a7c680f4bbf6227fce6bdf74d", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b192d63253e8dcc6221ef992ea9ebef7d3555166e8423aa5b553e86bc3c69a2"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, + "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, } diff --git a/test/bow/download_test.exs b/test/bow/download_test.exs index cba7c38..0a2e654 100644 --- a/test/bow/download_test.exs +++ b/test/bow/download_test.exs @@ -3,139 +3,83 @@ defmodule Bow.DownloadTest do @file_cat "test/files/cat.jpg" - import Bow.Download, only: [download: 2] - - setup do - client = - Tesla.client([Tesla.Middleware.FollowRedirects], fn env -> - case env do - %{url: "http://example.com/cat.png"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - %{url: "http://example.com/kitten.png"} -> - {:ok, %{env | status: 301, headers: [{"location", "http://example.com/cat.png"}]}} - - %{url: "http://example.com/loop.png"} -> - {:ok, %{env | status: 301, headers: [{"location", "http://example.com/loop.png"}]}} - - %{url: "http://example.com/notype.png"} -> - {:ok, %{env | status: 200, body: File.read!(@file_cat)}} - - %{url: "http://example.com/noext"} -> - {:ok, %{env | status: 200, body: File.read!(@file_cat)}} - - %{url: "http://example.com/u" <> _} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - %{url: "http://example.com"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - %{url: "http://example.com/dog.jpg"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "example/dog/nope"}] - }} - - %{url: "http://example.com/.weird-path"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - _ -> - {:ok, %{env | status: 404, body: "NotFound"}} - end - end) - - {:ok, client: client} + defp download(url, opts \\ []) do + Bow.Download.download(url, [downloader: Bow.TestDownloader] ++ opts) end - test "regular file", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/cat.png") + test "regular file" do + assert {:ok, file} = download("http://example.com/cat.png") assert file.name == "cat.png" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file with redirect", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/kitten.png") + test "file with redirect" do + assert {:ok, file} = download("http://example.com/kitten.png") assert file.name == "cat.png" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "too many redirects", %{client: client} do - assert {:error, %Tesla.Error{}} = download(client, "http://example.com/loop.png") + test "file with relative redirect" do + assert {:ok, file} = download("http://example.com/relative/kitten.png") + assert file.name == "cat.png" + end + + test "too many redirects" do + assert {:error, :too_many_redirects} = download("http://example.com/loop.png") + end + + test "max redirects option" do + assert {:error, :too_many_redirects} = + download("http://example.com/kitten.png", max_redirects: 0) end - test "file without content type", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/notype.png") + test "file without content type" do + assert {:ok, file} = download("http://example.com/notype.png") assert file.name == "notype.png" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file without extension", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/noext") + test "file without extension" do + assert {:ok, file} = download("http://example.com/noext") assert file.name == "noext" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file with invalid content type", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/dog.jpg") + test "file with invalid content type" do + assert {:ok, file} = download("http://example.com/dog.jpg") assert file.name == "dog.jpg" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "url with path starting with dot", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/.weird-path") + test "url with path starting with dot" do + assert {:ok, file} = download("http://example.com/.weird-path") assert Regex.match?(~r/.+\.png/, file.name) - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file not found", %{client: client} do - assert {:error, %{status: 404}} = download(client, "http://example.com/nope") + test "file not found" do + assert {:error, %{status: 404}} = download("http://example.com/nope") end - test "without file path", %{client: client} do - assert {:ok, file} = download(client, "http://example.com") - - uuid_name = file.name |> String.split(".") |> Enum.at(0) + test "downloader error" do + assert {:error, :econnrefused} = download("http://example.com/broken") + end - assert {:ok, _} = Ecto.UUID.dump(uuid_name) - assert file.path != nil + test "without file path" do + assert {:ok, file} = download("http://example.com") + assert file.name =~ ~r/^[0-9a-f]{32}\.png$/ assert File.read!(file.path) == File.read!(@file_cat) end - test "dynamic URL", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/u/91372?v=3&s=460") + test "dynamic URL" do + assert {:ok, file} = download("http://example.com/u/91372?v=3&s=460") assert file.name == "91372.png" end + + test "downloader from config" do + Application.put_env(:bow, :downloader, Bow.TestDownloader) + on_exit(fn -> Application.delete_env(:bow, :downloader) end) + + assert {:ok, %Bow{name: "cat.png"}} = Bow.Download.download("http://example.com/cat.png") + end end diff --git a/test/bow/downloader/httpc_test.exs b/test/bow/downloader/httpc_test.exs new file mode 100644 index 0000000..2782da3 --- /dev/null +++ b/test/bow/downloader/httpc_test.exs @@ -0,0 +1,111 @@ +defmodule Bow.Downloader.HttpcTest do + use ExUnit.Case + + alias Bow.Downloader.Httpc + + @file_cat "test/files/cat.jpg" + @big_size 3 * 1024 * 1024 + + defmodule Router do + use Plug.Router + + plug(:match) + plug(:dispatch) + + get "/cat.jpg" do + conn + |> put_resp_content_type("image/jpeg", nil) + |> send_file(200, "test/files/cat.jpg") + end + + get "/big" do + conn = send_chunked(conn, 200) + chunk = :binary.copy("a", 1024 * 1024) + Enum.reduce(1..3, conn, fn _, conn -> elem(chunk(conn, chunk), 1) end) + end + + get "/redirect" do + conn + |> put_resp_header("location", "/cat.jpg") + |> send_resp(302, "") + end + + get "/slow" do + Process.sleep(1000) + send_resp(conn, 200, "late") + end + + match _ do + send_resp(conn, 404, "nope") + end + end + + setup_all do + {:ok, server} = Bandit.start_link(plug: Router, ip: :loopback, port: 0, startup_log: false) + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + + %{base_url: "http://127.0.0.1:#{port}"} + end + + setup do + %{path: Plug.Upload.random_file!("httpc-test")} + end + + test "writes body to file", %{base_url: base_url, path: path} do + assert {:ok, 200, headers} = Httpc.get(base_url <> "/cat.jpg", path, []) + assert {"content-type", "image/jpeg"} in headers + assert File.read!(path) == File.read!(@file_cat) + end + + test "streams big body to file", %{base_url: base_url, path: path} do + assert {:ok, 200, _headers} = Httpc.get(base_url <> "/big", path, []) + assert File.stat!(path).size == @big_size + end + + test "does not follow redirects", %{base_url: base_url, path: path} do + assert {:ok, 302, headers} = Httpc.get(base_url <> "/redirect", path, []) + assert {"location", "/cat.jpg"} in headers + assert File.read!(path) == "" + end + + test "returns status of failed request", %{base_url: base_url, path: path} do + assert {:ok, 404, _headers} = Httpc.get(base_url <> "/nope", path, []) + assert File.read!(path) == "" + end + + test "timeout", %{base_url: base_url, path: path} do + assert {:error, :timeout} = Httpc.get(base_url <> "/slow", path, timeout: 100) + end + + test "connection error", %{path: path} do + assert {:error, {:failed_connect, _}} = Httpc.get("http://127.0.0.1:1/cat.jpg", path, []) + end + + test "default downloader of Bow.Download", %{base_url: base_url} do + assert {:ok, file} = Bow.Download.download(base_url <> "/redirect") + assert file.name == "cat.jpg" + assert File.read!(file.path) == File.read!(@file_cat) + end + + defmodule ReqDownloader do + # example from Bow.Downloader docs + @behaviour Bow.Downloader + + @impl true + def get(url, path, _opts) do + case Req.get(url, redirect: false, into: File.stream!(path)) do + {:ok, %{status: status, headers: headers}} -> + {:ok, status, for({name, values} <- headers, value <- values, do: {name, value})} + + {:error, reason} -> + {:error, reason} + end + end + end + + test "Req downloader example", %{base_url: base_url} do + assert {:ok, file} = Bow.Download.download(base_url <> "/redirect", downloader: ReqDownloader) + assert file.name == "cat.jpg" + assert File.read!(file.path) == File.read!(@file_cat) + end +end diff --git a/test/bow/ecto_test.exs b/test/bow/ecto_test.exs index 9d6d313..bd19880 100644 --- a/test/bow/ecto_test.exs +++ b/test/bow/ecto_test.exs @@ -363,58 +363,35 @@ defmodule Bow.EctoTest do end describe "Remote file URLs" do - defp client do - Tesla.client([], fn - %{url: "http://example.com/bear.png"} = env -> - {:ok, - %{ - env - | status: 200, - body: File.read!("test/files/bear.png"), - headers: [{"Content-Type", "image/png"}] - }} - - %{url: "http://example.com"} = env -> - {:ok, - %{ - env - | status: 200, - body: File.read!("test/files/bear.png"), - headers: [{"Content-Type", "image/png"}] - }} - - env -> - {:ok, %{env | status: 404}} - end) - end + @opts [downloader: Bow.TestDownloader] test "empty params" do params = %{} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert user.changes == %{} end test "empty string as param" do params = %{"remote_avatar_url" => ""} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert user.changes == %{} end test "invalid URL" do params = %{"remote_avatar_url" => "some-ribbish"} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert user.changes == %{} end test "valid URL" do params = %{"remote_avatar_url" => "http://example.com/bear.png"} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert %Bow{} = user.changes.avatar end test "valid URL without file path" do params = %{"remote_avatar_url" => "http://example.com"} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert %Bow{} = user.changes.avatar end end diff --git a/test/bow/exec_test.exs b/test/bow/exec_test.exs deleted file mode 100644 index d5ab7dc..0000000 --- a/test/bow/exec_test.exs +++ /dev/null @@ -1,52 +0,0 @@ -defmodule Bow.ExecTest do - use ExUnit.Case - - @file_cat "test/files/cat.jpg" - - import Bow.Exec, only: [exec: 3, exec: 4] - - setup do - source = Bow.new(path: @file_cat) - target = Bow.set(source, :name, "thumb_#{source.name}") - - {:ok, source: source, target: target} - end - - test "successful command", %{source: source, target: target} do - assert {:ok, %Bow{path: path}} = - exec(source, target, ["test/scripts/copy.sh", :input, :output]) - - assert path != nil - assert File.exists?(path) - end - - test "command not found", %{source: source, target: target} do - assert {:error, reason} = exec(source, target, ["test/scripts/notfound", :input, :output]) - assert reason[:output] =~ ~r/No such file/u - end - - test "failing command", %{source: source, target: target} do - assert {:error, reason} = exec(source, target, ["test/scripts/fail.sh", :input, :output]) - assert reason[:exit_code] != 0 - end - - test "timout", %{source: source, target: target} do - assert {:error, reason} = - exec(source, target, ["test/scripts/sleep.sh", :input, :output], timeout: 500) - - assert reason[:exit_code] == :timeout - end - - test "file name with quotes" do - file = "tmp/asdf'weird$name.pdf" - File.write!(file, "data") - source = Bow.new(path: file) - target = Bow.set(source, :name, "thumb_#{source.name}") - - assert {:ok, %Bow{path: path}} = - exec(source, target, ["test/scripts/copy.sh", :input, :output]) - - assert path != nil - assert File.exists?(path) - end -end diff --git a/test/bow/with_output_test.exs b/test/bow/with_output_test.exs new file mode 100644 index 0000000..03eac5d --- /dev/null +++ b/test/bow/with_output_test.exs @@ -0,0 +1,80 @@ +defmodule Bow.WithOutputTest do + use ExUnit.Case + + @file_cat "test/files/cat.jpg" + + setup do + File.write!("tmp/text.txt", "hello") + source = Bow.new(path: "tmp/text.txt") + target = Bow.set(source, :name, "upcase.md") + + {:ok, source: source, target: target} + end + + test "sets written file path on target", %{source: source, target: target} do + assert {:ok, %Bow{name: "upcase.md", path: path}} = + Bow.with_output(target, fn output_path -> + send(self(), {:output_path, output_path}) + File.write(output_path, source.path |> File.read!() |> String.upcase()) + end) + + assert File.read!(path) == "HELLO" + assert_received {:output_path, output_path} + assert Path.extname(output_path) == ".md" + refute File.exists?(output_path) + end + + test "returns error and removes partial output", %{target: target} do + assert {:error, :boom} = + Bow.with_output(target, fn output_path -> + File.write!(output_path, "partial") + send(self(), {:output_path, output_path}) + {:error, :boom} + end) + + assert_received {:output_path, output_path} + refute File.exists?(output_path) + end + + test "output not written", %{target: target} do + assert {:error, :output_not_found} = Bow.with_output(target, fn _output_path -> :ok end) + end + + test "invalid return value", %{target: target} do + assert_raise ArgumentError, ~r/expected :ok or {:error, reason}.*got: {"", 0}/, fn -> + Bow.with_output(target, fn _output_path -> {"", 0} end) + end + end + + test "external command", %{source: source, target: target} do + assert {:ok, %Bow{path: path}} = + Bow.with_output(target, fn output_path -> + case System.cmd("cp", [source.path, output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) + + assert File.read!(path) == "hello" + end + + test "works in uploader transform" do + defmodule ThumbUploader do + use Bow.Uploader + + def versions(_), do: [:original, :thumb] + + def transform(source, target, :thumb) do + Bow.with_output(target, &File.cp(source.path, &1)) + end + + def transform(source, target, version), do: super(source, target, version) + + def store_dir(_), do: "with_output" + end + + file = ThumbUploader.new(path: @file_cat) + assert {:ok, _} = Bow.store(file) + assert File.read!("tmp/bow/with_output/thumb_cat.jpg") == File.read!(@file_cat) + end +end diff --git a/test/bow_test.exs b/test/bow_test.exs index bc9452e..cc5d91d 100644 --- a/test/bow_test.exs +++ b/test/bow_test.exs @@ -424,7 +424,7 @@ defmodule BowTest do def versions(_), do: [:original, :pdf] def transform(source, target, :original), do: transform_original(source, target) - def transform(_source, _target, :pdf), do: %{}.foo + 1 + def transform(_source, _target, :pdf), do: raise(KeyError, key: :foo, term: %{}) def store_dir(_), do: "raise" end diff --git a/test/scripts/copy.sh b/test/scripts/copy.sh deleted file mode 100755 index 2f2e8ab..0000000 --- a/test/scripts/copy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -e -cp "$1" "$2" diff --git a/test/scripts/fail.sh b/test/scripts/fail.sh deleted file mode 100755 index b9e71c6..0000000 --- a/test/scripts/fail.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exit 5 diff --git a/test/scripts/sleep.sh b/test/scripts/sleep.sh deleted file mode 100755 index 9eaac3d..0000000 --- a/test/scripts/sleep.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -sleep 10000000 diff --git a/test/support/test_downloader.ex b/test/support/test_downloader.ex new file mode 100644 index 0000000..bc7bcc9 --- /dev/null +++ b/test/support/test_downloader.ex @@ -0,0 +1,33 @@ +defmodule Bow.TestDownloader do + @moduledoc false + @behaviour Bow.Downloader + + @file_cat "test/files/cat.jpg" + @file_bear "test/files/bear.png" + + @impl true + def get(url, path, _opts) do + case url do + "http://example.com/cat.png" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com/kitten.png" -> redirect("http://example.com/cat.png") + "http://example.com/relative/kitten.png" -> redirect("../cat.png") + "http://example.com/loop.png" -> redirect("http://example.com/loop.png") + "http://example.com/notype.png" -> ok(path, @file_cat, []) + "http://example.com/noext" -> ok(path, @file_cat, []) + "http://example.com/u" <> _ -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com/dog.jpg" -> ok(path, @file_cat, [{"content-type", "example/dog/nope"}]) + "http://example.com/.weird-path" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com/bear.png" -> ok(path, @file_bear, [{"content-type", "image/png"}]) + "http://example.com/broken" -> {:error, :econnrefused} + _ -> {:ok, 404, []} + end + end + + defp ok(path, file, headers) do + File.cp!(file, path) + {:ok, 200, headers} + end + + defp redirect(location), do: {:ok, 301, [{"location", location}]} +end From d4387f603b434e5ab034b561be465ee5e5835355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Dro=C5=BCd=C5=BCy=C5=84ski?= Date: Thu, 24 Sep 2026 14:36:17 +0200 Subject: [PATCH 2/5] Create tmp directory in test helper Tests write to tmp/, which does not exist in a fresh checkout. --- test/test_helper.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_helper.exs b/test/test_helper.exs index 788f6ec..ab0f4ce 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,3 @@ {:ok, _} = Application.ensure_all_started(:ecto_sql) +File.mkdir_p!("tmp") ExUnit.start(exclude: [:s3, :ecto]) From 65447b970cfabcabedcdef899584230aa7178073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Dro=C5=BCd=C5=BCy=C5=84ski?= Date: Tue, 29 Sep 2026 08:59:31 +0200 Subject: [PATCH 3/5] Address downloader review feedback - Strip credentials and origin bound headers on cross-origin redirects (RFC 9110 section 15.4, same as Tesla FollowRedirects) - Run :httpc request in a Task so late messages never reach the caller - Make :timeout the total download time instead of idle time - Add :headers and :max_size options, remove temp file on download error - Document SSRF risk of user provided URLs and ExAws 2.7 hackney HEAD issue --- CHANGELOG.md | 9 ++- README.md | 29 +++++++-- lib/bow/download.ex | 48 +++++++++++++- lib/bow/downloader.ex | 17 +++-- lib/bow/downloader/httpc.ex | 100 +++++++++++++++++++++++------ lib/bow/ecto.ex | 9 ++- test/bow/download_test.exs | 47 ++++++++++++++ test/bow/downloader/httpc_test.exs | 76 +++++++++++++++++++++- test/support/test_downloader.ex | 6 +- 9 files changed, 307 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa02550..53e15ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,19 @@ Breaking changes: * `Bow.Download.download(client, url)` is now `Bow.Download.download(url, opts)` * `Bow.Ecto.cast_uploads/4` and `Bow.Ecto.download_params/3` take options (e.g. `downloader: ...`) instead of a Tesla client -* `Bow.Download.download/2` errors are `{:error, %{status: status, headers: headers}}` - and `{:error, :too_many_redirects}` +* `Bow.Download.download/2` errors are `{:error, %{status: status, headers: headers}}`, + `{:error, :too_many_redirects}`, `{:error, :max_size_exceeded}` and `{:error, :timeout}` +* Default downloader sends `user-agent: bow`, override it with the `:headers` option Other changes: * Updated dependencies (Ecto 3.10+, Plug 1.14+, ExAws 2.4+) * Add `Bow.with_output/2` helper for writing processed files * Downloaded files are streamed to disk instead of being loaded into memory +* `Bow.Download.download/2` and `Bow.Ecto.cast_uploads/4` options: `:headers`, `:max_size` (file size limit) + and `:timeout` (total download time) +* Credentials (`authorization`, `cookie`, ...) are removed from `:headers` on redirect to a different origin +* Document SSRF risk of downloading user provided URLs and the ExAws 2.7 hackney `HEAD` issue when loading files from `Bow.Storage.S3` * fix: `Bow.Download` no longer requires Ecto # v0.4.3 diff --git a/README.md b/README.md index b3d78c9..1bc5c3a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,11 @@ def deps do end ``` +> **Note** +> Loading files with `Bow.Storage.S3` uses `ExAws.S3.download_file/4`, which sends a `HEAD` request. +> The hackney client of ExAws 2.7 crashes on `HEAD` responses, use the Req client instead: +> `config :ex_aws, http_client: ExAws.Request.Req` (requires `{:req, "~> 0.5"}`). + Bow requires Elixir 1.16 or newer. ## Usage @@ -192,16 +197,32 @@ end ### Downloading remote files `Bow.Ecto.cast_uploads/4` downloads files given as `remote__url` params -(e.g. `remote_avatar_url`). Files are downloaded with Erlang `:httpc` by default, -you can use any HTTP client by implementing the `Bow.Downloader` behaviour: +(e.g. `remote_avatar_url`). Files are streamed to disk with Erlang `:httpc` by default. + +```elixir +changeset +|> Bow.Ecto.cast_uploads(params, [:avatar], + max_size: 10_000_000, + timeout: 15_000, + headers: [{"user-agent", "my-app"}] +) +``` + +On redirect to a different origin, credentials like `authorization` and `cookie` are removed from `:headers`. + +> **Warning** +> The URL comes from users, so it can point to your internal services, e.g. the +> `http://169.254.169.254/` cloud metadata endpoint (SSRF). Validate the URL before downloading +> it and always set `:max_size` to protect the disk. + +You can use any HTTP client by implementing the `Bow.Downloader` behaviour +(see its docs for an example based on Req): ```elixir # config/config.exs config :bow, downloader: MyApp.BowDownloader ``` -See `Bow.Downloader` docs for an example based on Req. - ### Processing files `transform/3` can generate the version in any way you like. `Bow.with_output/2` helps with diff --git a/lib/bow/download.ex b/lib/bow/download.ex index dd59954..45b4bcb 100644 --- a/lib/bow/download.ex +++ b/lib/bow/download.ex @@ -2,16 +2,34 @@ defmodule Bow.Download do @max_redirects 5 @redirect_statuses [301, 302, 303, 307, 308] + # Headers filtering on redirect, per RFC 9110 section 15.4 (same lists as Tesla.Middleware.FollowRedirects). + # Hop-by-hop and conditional request headers do not carry over to a new request. + @always_strip ~w(connection keep-alive proxy-connection te trailer transfer-encoding upgrade + if-match if-modified-since if-none-match if-range if-unmodified-since) + # Credentials and origin bound headers must not leak to a different origin. + @cross_origin_strip ~w(authorization cookie host origin proxy-authorization referer) + @doc """ Download file from given URL The file name is taken from the URL (after following redirects) with extension based on the `content-type` header. + > #### Untrusted URLs {: .warning} + > + > When the URL comes from users (e.g. `remote_avatar_url` params), it can point to + > internal services, like `http://169.254.169.254/` cloud metadata endpoint (SSRF). + > Validate the URL before downloading it and set `:max_size`. + Options: - `:downloader` - module implementing `Bow.Downloader`, defaults to `:downloader` config or `Bow.Downloader.Httpc` - `:max_redirects` - maximum number of redirects to follow, defaults to `#{@max_redirects}` + - `:headers` - request headers as a list of `{name, value}` tuples. On redirect to a different + origin (scheme, host or port) credentials like `authorization` and `cookie` are removed. + - `:max_size` - maximum file size in bytes. The downloader should stop the download when it's + exceeded, Bow checks the downloaded file size as well. Returns `{:error, :max_size_exceeded}`. + - `:timeout` - download timeout in milliseconds, see the downloader docs for the default Other options are passed to the downloader. """ @@ -21,8 +39,13 @@ defmodule Bow.Download do {max_redirects, opts} = Keyword.pop(opts, :max_redirects, @max_redirects) path = Plug.Upload.random_file!("bow-download") - with {:ok, url, headers} <- get(downloader, encode(url), path, opts, max_redirects) do + with {:ok, url, headers} <- get(downloader, encode(url), path, opts, max_redirects), + :ok <- check_size(path, opts[:max_size]) do {:ok, Bow.new(name: name(url, headers), path: path)} + else + {:error, reason} -> + File.rm(path) + {:error, reason} end end @@ -50,12 +73,35 @@ defmodule Bow.Download do defp follow_redirect(downloader, url, path, opts, redirects_left, status, headers) do with {_, location} <- List.keyfind(headers, "location", 0) do next_url = url |> URI.merge(location) |> to_string() + opts = Keyword.update(opts, :headers, [], &filter_headers(&1, url, next_url)) get(downloader, next_url, path, opts, redirects_left - 1) else nil -> {:error, %{status: status, headers: headers}} end end + defp filter_headers(headers, url, next_url) do + drop = + if cross_origin?(url, next_url), + do: @always_strip ++ @cross_origin_strip, + else: @always_strip + + Enum.reject(headers, fn {name, _value} -> String.downcase(to_string(name)) in drop end) + end + + defp cross_origin?(url, next_url), do: origin(url) != origin(next_url) + + defp origin(url) do + uri = URI.parse(url) + {uri.scheme, uri.host && String.downcase(uri.host), uri.port} + end + + defp check_size(_path, nil), do: :ok + + defp check_size(path, max_size) do + if File.stat!(path).size > max_size, do: {:error, :max_size_exceeded}, else: :ok + end + defp name(url, headers) do base = case URI.parse(url) do diff --git a/lib/bow/downloader.ex b/lib/bow/downloader.ex index 77b359c..552b5af 100644 --- a/lib/bow/downloader.ex +++ b/lib/bow/downloader.ex @@ -19,8 +19,10 @@ defmodule Bow.Downloader do @behaviour Bow.Downloader @impl true - def get(url, path, _opts) do - case Req.get(url, redirect: false, into: File.stream!(path)) do + def get(url, path, opts) do + req_opts = [redirect: false, into: File.stream!(path), headers: Keyword.get(opts, :headers, [])] + + case Req.get(url, req_opts) do {:ok, %{status: status, headers: headers}} -> {:ok, status, for({name, values} <- headers, value <- values, do: {name, value})} @@ -39,9 +41,16 @@ defmodule Bow.Downloader do Arguments: - `url` - URL to request - `path` - path the response body must be written to when the status is 2xx - - `opts` - options given to `Bow.Download.download/2` + - `opts` - options given to `Bow.Download.download/2`, the downloader should support: + - `:headers` - request headers as a list of `{name, value}` tuples + - `:max_size` - maximum body size in bytes, stop the download and return + `{:error, :max_size_exceeded}` when it's exceeded + - `:timeout` - total download time in milliseconds + + Must not follow redirects, `Bow.Download` takes care of them and calls the downloader again + with the next URL. Options may differ between calls, e.g. credentials in `:headers` are removed + on redirect to a different origin, so always use the given `opts` instead of static configuration. - Must not follow redirects, `Bow.Download` takes care of them. Must return the response status and headers with lowercase names. """ @callback get(url :: String.t(), path :: Path.t(), opts :: keyword) :: diff --git a/lib/bow/downloader/httpc.ex b/lib/bow/downloader/httpc.ex index c87c83f..ae729c7 100644 --- a/lib/bow/downloader/httpc.ex +++ b/lib/bow/downloader/httpc.ex @@ -4,19 +4,45 @@ defmodule Bow.Downloader.Httpc do The response body is streamed directly to the file. HTTPS certificates are verified using the operating system CA store. + The request runs in a separate process, so late `:httpc` messages + (e.g. after a timeout) never reach the calling process. Options: - - `:timeout` - request timeout in milliseconds, defaults to `30_000` + - `:timeout` - total download time in milliseconds (not the idle time between chunks), + defaults to `30_000`. Returns `{:error, :timeout}` when exceeded. + - `:headers` - request headers as a list of `{name, value}` tuples, + `user-agent` defaults to `"bow"` + - `:max_size` - maximum body size in bytes. The download is aborted as soon as + `content-length` or the received data exceeds it. Returns `{:error, :max_size_exceeded}`. """ @behaviour Bow.Downloader @default_timeout 30_000 + @default_headers [{"user-agent", "bow"}] + # request/4 stops itself at the deadline, this only covers the cleanup + @shutdown_margin 1_000 @impl true def get(url, path, opts) do timeout = Keyword.get(opts, :timeout, @default_timeout) - request = {String.to_charlist(url), [{~c"user-agent", ~c"bow"}]} + task = Task.async(fn -> request(url, path, opts, timeout) end) + + case Task.yield(task, timeout + @shutdown_margin) || Task.shutdown(task, :brutal_kill) do + {:ok, result} -> result + {:exit, reason} -> {:error, reason} + nil -> {:error, :timeout} + end + end + + defp request(url, path, opts, timeout) do + state = %{ + path: path, + max_size: opts[:max_size], + deadline: System.monotonic_time(:millisecond) + timeout + } + + request = {String.to_charlist(url), request_headers(opts)} http_opts = [ timeout: timeout, @@ -30,49 +56,51 @@ defmodule Bow.Downloader.Httpc do stream: :self, body_format: :binary ) do - {:ok, ref} -> receive_response(ref, path, timeout) + {:ok, ref} -> receive_response(ref, state) {:error, reason} -> {:error, reason} end end # :httpc streams only 200 and 206 responses, others are received at once - defp receive_response(ref, path, timeout) do + defp receive_response(ref, state) do receive do {:http, {^ref, :stream_start, headers}} -> - save_stream(ref, path, headers, timeout) + save_stream(ref, normalize(headers), state) {:http, {^ref, {{_version, status, _reason}, headers, body}}} -> - save_body(path, status, headers, body) + save_body(status, normalize(headers), body, state) {:http, {^ref, {:error, reason}}} -> {:error, reason} after - timeout -> cancel(ref, :timeout) + remaining(state) -> cancel(ref, :timeout) end end - defp save_stream(ref, path, headers, timeout) do - with {:ok, :ok} <- File.open(path, [:write, :binary], &stream_body(ref, &1, timeout)) do - {:ok, 200, normalize(headers)} + defp save_stream(ref, headers, state) do + with :ok <- check_content_length(headers, state.max_size), + {:ok, :ok} <- File.open(state.path, [:write, :binary], &stream_body(ref, &1, 0, state)) do + # Range header is never sent, so a streamed response can't be 206 + {:ok, 200, headers} else {:ok, {:error, reason}} -> {:error, reason} {:error, reason} -> cancel(ref, reason) end end - defp save_body(path, status, headers, body) when status in 200..299 do - with :ok <- File.write(path, body) do - {:ok, status, normalize(headers)} + defp save_body(status, headers, body, state) when status in 200..299 do + with :ok <- check_size(byte_size(body), state.max_size), + :ok <- File.write(state.path, body) do + {:ok, status, headers} end end - defp save_body(_path, status, headers, _body), do: {:ok, status, normalize(headers)} + defp save_body(status, headers, _body, _state), do: {:ok, status, headers} - defp stream_body(ref, file, timeout) do + defp stream_body(ref, file, size, state) do receive do {:http, {^ref, :stream, chunk}} -> - IO.binwrite(file, chunk) - stream_body(ref, file, timeout) + write_chunk(ref, file, size + byte_size(chunk), chunk, state) {:http, {^ref, :stream_end, _headers}} -> :ok @@ -80,15 +108,51 @@ defmodule Bow.Downloader.Httpc do {:http, {^ref, {:error, reason}}} -> {:error, reason} after - timeout -> cancel(ref, :timeout) + remaining(state) -> cancel(ref, :timeout) + end + end + + defp write_chunk(ref, _file, size, _chunk, %{max_size: max_size}) + when is_integer(max_size) and size > max_size do + cancel(ref, :max_size_exceeded) + end + + defp write_chunk(ref, file, size, chunk, state) do + IO.binwrite(file, chunk) + stream_body(ref, file, size, state) + end + + defp check_content_length(headers, max_size) do + with {_, value} <- List.keyfind(headers, "content-length", 0), + {length, ""} <- Integer.parse(value) do + check_size(length, max_size) + else + _ -> :ok end end + defp check_size(size, max_size) when is_integer(max_size) and size > max_size, + do: {:error, :max_size_exceeded} + + defp check_size(_size, _max_size), do: :ok + + defp remaining(state), do: max(state.deadline - System.monotonic_time(:millisecond), 0) + defp cancel(ref, reason) do :httpc.cancel_request(ref) {:error, reason} end + defp request_headers(opts) do + headers = Keyword.get(opts, :headers, []) + names = MapSet.new(headers, fn {name, _value} -> String.downcase(to_string(name)) end) + defaults = Enum.reject(@default_headers, fn {name, _value} -> name in names end) + + for {name, value} <- defaults ++ headers do + {to_charlist(name), to_charlist(value)} + end + end + defp normalize(headers) do for {name, value} <- headers do {name |> to_string() |> String.downcase(), to_string(value)} diff --git a/lib/bow/ecto.ex b/lib/bow/ecto.ex index 9d7fd8d..6d6b2cc 100644 --- a/lib/bow/ecto.ex +++ b/lib/bow/ecto.ex @@ -303,12 +303,17 @@ defmodule Bow.Ecto do Download remote files for given fields, i.e. `params["remote_avatar_url"] = "http://example.com/some/file.png"` - Options are passed to `Bow.Download.download/2`, e.g. `:downloader`. + Options are passed to `Bow.Download.download/2`, e.g. `:max_size`, `:timeout` or `:headers`. + + > #### Untrusted URLs {: .warning} + > + > The URLs come from users, see `Bow.Download.download/2` about validating them + > and limiting the file size. Example changeset |> cast(params, [:name, :avatar]) - |> Bow.Ecto.cast_uploads(params, [:avatar]) + |> Bow.Ecto.cast_uploads(params, [:avatar], max_size: 10_000_000) """ @spec cast_uploads(any, map, list, keyword) :: Ecto.Changeset.t() def cast_uploads(changeset, params, fields, opts \\ []) do diff --git a/test/bow/download_test.exs b/test/bow/download_test.exs index 0a2e654..6ac9e35 100644 --- a/test/bow/download_test.exs +++ b/test/bow/download_test.exs @@ -82,4 +82,51 @@ defmodule Bow.DownloadTest do assert {:ok, %Bow{name: "cat.png"}} = Bow.Download.download("http://example.com/cat.png") end + + describe "headers on redirect" do + @headers [ + {"Authorization", "Bearer secret"}, + {"cookie", "session=1"}, + {"If-None-Match", "etag"}, + {"accept", "image/*"} + ] + + test "are passed to the downloader" do + assert {:ok, _} = download("http://example.com/cat.png", headers: @headers) + assert_received {:downloader_get, _url, _path, opts} + assert opts[:headers] == @headers + end + + test "keep credentials on the same origin" do + assert {:ok, _} = download("http://example.com/kitten.png", headers: @headers) + assert_received {:downloader_get, "http://example.com/kitten.png", _path, _opts} + assert_received {:downloader_get, "http://example.com/cat.png", _path, opts} + + assert opts[:headers] == [ + {"Authorization", "Bearer secret"}, + {"cookie", "session=1"}, + {"accept", "image/*"} + ] + end + + test "remove credentials on a different origin" do + assert {:ok, file} = download("http://example.com/to-cdn.png", headers: @headers) + assert file.name == "cat.png" + assert_received {:downloader_get, "http://example.com/to-cdn.png", _path, _opts} + assert_received {:downloader_get, "https://cdn.example.org/cat.png", _path, opts} + assert opts[:headers] == [{"accept", "image/*"}] + end + end + + test "max size is checked after download" do + assert {:error, :max_size_exceeded} = download("http://example.com/cat.png", max_size: 10) + assert_received {:downloader_get, _url, path, _opts} + refute File.exists?(path) + end + + test "removes file on error" do + assert {:error, %{status: 404}} = download("http://example.com/nope") + assert_received {:downloader_get, _url, path, _opts} + refute File.exists?(path) + end end diff --git a/test/bow/downloader/httpc_test.exs b/test/bow/downloader/httpc_test.exs index 2782da3..79c844e 100644 --- a/test/bow/downloader/httpc_test.exs +++ b/test/bow/downloader/httpc_test.exs @@ -35,6 +35,28 @@ defmodule Bow.Downloader.HttpcTest do send_resp(conn, 200, "late") end + # sends a chunk every 100ms for 1s, so it's never idle longer than the timeout + get "/drip" do + conn = send_chunked(conn, 200) + + # stop when the client disconnects after its timeout + Enum.reduce_while(1..10, conn, fn _, conn -> + Process.sleep(100) + + case chunk(conn, ".") do + {:ok, conn} -> {:cont, conn} + {:error, :closed} -> {:halt, conn} + end + end) + end + + get "/headers" do + body = + Enum.map_join(conn.req_headers, "\n", fn {name, value} -> "#{name}: #{value}" end) + + send_resp(conn, 200, body) + end + match _ do send_resp(conn, 404, "nope") end @@ -77,6 +99,50 @@ defmodule Bow.Downloader.HttpcTest do assert {:error, :timeout} = Httpc.get(base_url <> "/slow", path, timeout: 100) end + test "timeout is the total download time", %{base_url: base_url, path: path} do + started_at = System.monotonic_time(:millisecond) + assert {:error, :timeout} = Httpc.get(base_url <> "/drip", path, timeout: 300) + assert System.monotonic_time(:millisecond) - started_at < 800 + end + + test "late messages do not reach the caller", %{base_url: base_url, path: path} do + assert {:error, :timeout} = Httpc.get(base_url <> "/drip", path, timeout: 150) + assert {:error, :timeout} = Httpc.get(base_url <> "/slow", path, timeout: 100) + refute_receive {:http, _}, 1_500 + end + + test "default user agent", %{base_url: base_url, path: path} do + assert {:ok, 200, _headers} = Httpc.get(base_url <> "/headers", path, []) + assert File.read!(path) =~ "user-agent: bow" + end + + test "custom headers", %{base_url: base_url, path: path} do + headers = [{"User-Agent", "my-app"}, {"x-token", "secret"}] + assert {:ok, 200, _headers} = Httpc.get(base_url <> "/headers", path, headers: headers) + + body = File.read!(path) + assert body =~ "user-agent: my-app" + refute body =~ "user-agent: bow" + assert body =~ "x-token: secret" + end + + test "max size exceeded by content-length", %{base_url: base_url, path: path} do + assert {:error, :max_size_exceeded} = Httpc.get(base_url <> "/cat.jpg", path, max_size: 100) + assert File.read!(path) == "" + end + + test "max size exceeded while streaming", %{base_url: base_url, path: path} do + assert {:error, :max_size_exceeded} = + Httpc.get(base_url <> "/big", path, max_size: 1024 * 1024) + + assert File.stat!(path).size <= 1024 * 1024 + end + + test "max size not exceeded", %{base_url: base_url, path: path} do + size = File.stat!(@file_cat).size + assert {:ok, 200, _headers} = Httpc.get(base_url <> "/cat.jpg", path, max_size: size) + end + test "connection error", %{path: path} do assert {:error, {:failed_connect, _}} = Httpc.get("http://127.0.0.1:1/cat.jpg", path, []) end @@ -92,8 +158,14 @@ defmodule Bow.Downloader.HttpcTest do @behaviour Bow.Downloader @impl true - def get(url, path, _opts) do - case Req.get(url, redirect: false, into: File.stream!(path)) do + def get(url, path, opts) do + req_opts = [ + redirect: false, + into: File.stream!(path), + headers: Keyword.get(opts, :headers, []) + ] + + case Req.get(url, req_opts) do {:ok, %{status: status, headers: headers}} -> {:ok, status, for({name, values} <- headers, value <- values, do: {name, value})} diff --git a/test/support/test_downloader.ex b/test/support/test_downloader.ex index bc7bcc9..dcdd8ef 100644 --- a/test/support/test_downloader.ex +++ b/test/support/test_downloader.ex @@ -6,12 +6,16 @@ defmodule Bow.TestDownloader do @file_bear "test/files/bear.png" @impl true - def get(url, path, _opts) do + def get(url, path, opts) do + send(self(), {:downloader_get, url, path, opts}) + case url do "http://example.com/cat.png" -> ok(path, @file_cat, [{"content-type", "image/png"}]) "http://example.com/kitten.png" -> redirect("http://example.com/cat.png") "http://example.com/relative/kitten.png" -> redirect("../cat.png") "http://example.com/loop.png" -> redirect("http://example.com/loop.png") + "http://example.com/to-cdn.png" -> redirect("https://cdn.example.org/cat.png") + "https://cdn.example.org/cat.png" -> ok(path, @file_cat, [{"content-type", "image/png"}]) "http://example.com/notype.png" -> ok(path, @file_cat, []) "http://example.com/noext" -> ok(path, @file_cat, []) "http://example.com/u" <> _ -> ok(path, @file_cat, [{"content-type", "image/png"}]) From 1bb41e31dfe40ab93bbbf2765280e554b4f6d17e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Dro=C5=BCd=C5=BCy=C5=84ski?= Date: Tue, 29 Sep 2026 10:19:40 +0200 Subject: [PATCH 4/5] Clean up with_output on raise and handle chunk write errors Wrap with_output/2 function call in try/after so output written before an exception does not stay in the tmp dir. Cancel the download when IO.binwrite fails instead of returning a truncated file. --- lib/bow.ex | 24 +++++++++++++----------- lib/bow/downloader/httpc.ex | 7 +++++-- test/bow/with_output_test.exs | 13 +++++++++++++ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/lib/bow.ex b/lib/bow.ex index 8473096..8725188 100644 --- a/lib/bow.ex +++ b/lib/bow.ex @@ -145,19 +145,21 @@ defmodule Bow do # so the file is still removed together with the process output_path = tmp_path <> target.ext - case fun.(output_path) do - :ok -> - keep_output(target, output_path, tmp_path) - - {:error, reason} -> - File.rm(output_path) - {:error, reason} + try do + case fun.(output_path) do + :ok -> + keep_output(target, output_path, tmp_path) - other -> - File.rm(output_path) + {:error, reason} -> + {:error, reason} - raise ArgumentError, - "expected :ok or {:error, reason} from with_output/2 function, got: #{inspect(other)}" + other -> + raise ArgumentError, + "expected :ok or {:error, reason} from with_output/2 function, got: #{inspect(other)}" + end + after + # no-op after a successful rename, removes partial output otherwise + File.rm(output_path) end end diff --git a/lib/bow/downloader/httpc.ex b/lib/bow/downloader/httpc.ex index ae729c7..50c8f17 100644 --- a/lib/bow/downloader/httpc.ex +++ b/lib/bow/downloader/httpc.ex @@ -118,8 +118,11 @@ defmodule Bow.Downloader.Httpc do end defp write_chunk(ref, file, size, chunk, state) do - IO.binwrite(file, chunk) - stream_body(ref, file, size, state) + with :ok <- IO.binwrite(file, chunk) do + stream_body(ref, file, size, state) + else + {:error, reason} -> cancel(ref, reason) + end end defp check_content_length(headers, max_size) do diff --git a/test/bow/with_output_test.exs b/test/bow/with_output_test.exs index 03eac5d..5ec9a45 100644 --- a/test/bow/with_output_test.exs +++ b/test/bow/with_output_test.exs @@ -36,6 +36,19 @@ defmodule Bow.WithOutputTest do refute File.exists?(output_path) end + test "removes partial output when function raises", %{target: target} do + assert_raise RuntimeError, "boom", fn -> + Bow.with_output(target, fn output_path -> + File.write!(output_path, "partial") + send(self(), {:output_path, output_path}) + raise "boom" + end) + end + + assert_received {:output_path, output_path} + refute File.exists?(output_path) + end + test "output not written", %{target: target} do assert {:error, :output_not_found} = Bow.with_output(target, fn _output_path -> :ok end) end From 2d4e6cdf57410e96aa22299db78eede41cd39576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krystian=20Dro=C5=BCd=C5=BCy=C5=84ski?= Date: Tue, 29 Sep 2026 11:45:31 +0200 Subject: [PATCH 5/5] Move redirect handling to Bow.Downloader implementations Downloaders now follow redirects themselves and return the final URL, so clients like Req or Tesla can use their own redirect support. Bow.Downloader.Httpc gets the redirect loop, :max_redirects and cross-origin header filtering, with one deadline for all hops. Also use :file.write/2 for streamed chunks, IO.binwrite/2 raises instead of returning an error tuple. --- CHANGELOG.md | 6 +- README.md | 6 +- lib/bow/download.ex | 72 ++-------------- lib/bow/downloader.ex | 49 ++++++++--- lib/bow/downloader/httpc.ex | 76 +++++++++++++++-- test/bow/download_test.exs | 58 +++---------- test/bow/downloader/httpc_test.exs | 133 ++++++++++++++++++++++++----- test/support/test_downloader.ex | 19 +++-- 8 files changed, 252 insertions(+), 167 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e15ad..280a033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ Breaking changes: `Bow.with_output/2` and any tool you like (`System.cmd/3`, Vix, MuonTrap, erlexec...). `:exec_timeout` config removed * `tesla` dependency removed. Remote files are downloaded with `:httpc` by default (`Bow.Downloader.Httpc`), - use `config :bow, downloader: MyDownloader` to plug in your own `Bow.Downloader` (e.g. Req or Tesla) + use `config :bow, downloader: MyDownloader` to plug in your own `Bow.Downloader` (e.g. Req or Tesla). + Downloaders follow redirects themselves and return the final URL used for the file name * `Bow.Download.download(client, url)` is now `Bow.Download.download(url, opts)` * `Bow.Ecto.cast_uploads/4` and `Bow.Ecto.download_params/3` take options (e.g. `downloader: ...`) instead of a Tesla client @@ -24,7 +25,8 @@ Other changes: * Downloaded files are streamed to disk instead of being loaded into memory * `Bow.Download.download/2` and `Bow.Ecto.cast_uploads/4` options: `:headers`, `:max_size` (file size limit) and `:timeout` (total download time) -* Credentials (`authorization`, `cookie`, ...) are removed from `:headers` on redirect to a different origin +* `Bow.Downloader.Httpc` removes credentials (`authorization`, `cookie`, ...) from `:headers` on redirect + to a different origin, `:max_redirects` option (defaults to 5) * Document SSRF risk of downloading user provided URLs and the ExAws 2.7 hackney `HEAD` issue when loading files from `Bow.Storage.S3` * fix: `Bow.Download` no longer requires Ecto diff --git a/README.md b/README.md index 1bc5c3a..0c4d2ac 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,8 @@ changeset ) ``` -On redirect to a different origin, credentials like `authorization` and `cookie` are removed from `:headers`. +The default downloader follows up to 5 redirects (`:max_redirects` option). On redirect to a different +origin, credentials like `authorization` and `cookie` are removed from `:headers`. > **Warning** > The URL comes from users, so it can point to your internal services, e.g. the @@ -216,7 +217,8 @@ On redirect to a different origin, credentials like `authorization` and `cookie` > it and always set `:max_size` to protect the disk. You can use any HTTP client by implementing the `Bow.Downloader` behaviour -(see its docs for an example based on Req): +(see its docs for an example based on Req). Your downloader is then responsible for following +redirects without leaking credentials to a different origin: ```elixir # config/config.exs diff --git a/lib/bow/download.ex b/lib/bow/download.ex index 45b4bcb..bb87131 100644 --- a/lib/bow/download.ex +++ b/lib/bow/download.ex @@ -1,19 +1,9 @@ defmodule Bow.Download do - @max_redirects 5 - @redirect_statuses [301, 302, 303, 307, 308] - - # Headers filtering on redirect, per RFC 9110 section 15.4 (same lists as Tesla.Middleware.FollowRedirects). - # Hop-by-hop and conditional request headers do not carry over to a new request. - @always_strip ~w(connection keep-alive proxy-connection te trailer transfer-encoding upgrade - if-match if-modified-since if-none-match if-range if-unmodified-since) - # Credentials and origin bound headers must not leak to a different origin. - @cross_origin_strip ~w(authorization cookie host origin proxy-authorization referer) - @doc """ Download file from given URL - The file name is taken from the URL (after following redirects) - with extension based on the `content-type` header. + The file name is taken from the final URL returned by the downloader (after redirects), + or the requested URL, with extension based on the `content-type` header. > #### Untrusted URLs {: .warning} > @@ -24,24 +14,23 @@ defmodule Bow.Download do Options: - `:downloader` - module implementing `Bow.Downloader`, defaults to `:downloader` config or `Bow.Downloader.Httpc` - - `:max_redirects` - maximum number of redirects to follow, defaults to `#{@max_redirects}` - - `:headers` - request headers as a list of `{name, value}` tuples. On redirect to a different - origin (scheme, host or port) credentials like `authorization` and `cookie` are removed. + - `:headers` - request headers as a list of `{name, value}` tuples - `:max_size` - maximum file size in bytes. The downloader should stop the download when it's exceeded, Bow checks the downloaded file size as well. Returns `{:error, :max_size_exceeded}`. - `:timeout` - download timeout in milliseconds, see the downloader docs for the default - Other options are passed to the downloader. + Other options (e.g. `:max_redirects`) are passed to the downloader. Redirects are followed + by the downloader, see `Bow.Downloader.Httpc` for the default behaviour. """ @spec download(url :: String.t(), opts :: keyword) :: {:ok, Bow.t()} | {:error, any} def download(url, opts \\ []) do {downloader, opts} = Keyword.pop(opts, :downloader, downloader()) - {max_redirects, opts} = Keyword.pop(opts, :max_redirects, @max_redirects) + url = encode(url) path = Plug.Upload.random_file!("bow-download") - with {:ok, url, headers} <- get(downloader, encode(url), path, opts, max_redirects), + with {:ok, response} <- downloader.get(url, path, opts), :ok <- check_size(path, opts[:max_size]) do - {:ok, Bow.new(name: name(url, headers), path: path)} + {:ok, Bow.new(name: name(response[:url] || url, response.headers), path: path)} else {:error, reason} -> File.rm(path) @@ -51,51 +40,6 @@ defmodule Bow.Download do defp downloader, do: Application.get_env(:bow, :downloader, Bow.Downloader.Httpc) - defp get(downloader, url, path, opts, redirects_left) do - case downloader.get(url, path, opts) do - {:ok, 200, headers} -> - {:ok, url, headers} - - {:ok, status, _headers} when status in @redirect_statuses and redirects_left == 0 -> - {:error, :too_many_redirects} - - {:ok, status, headers} when status in @redirect_statuses -> - follow_redirect(downloader, url, path, opts, redirects_left, status, headers) - - {:ok, status, headers} -> - {:error, %{status: status, headers: headers}} - - {:error, reason} -> - {:error, reason} - end - end - - defp follow_redirect(downloader, url, path, opts, redirects_left, status, headers) do - with {_, location} <- List.keyfind(headers, "location", 0) do - next_url = url |> URI.merge(location) |> to_string() - opts = Keyword.update(opts, :headers, [], &filter_headers(&1, url, next_url)) - get(downloader, next_url, path, opts, redirects_left - 1) - else - nil -> {:error, %{status: status, headers: headers}} - end - end - - defp filter_headers(headers, url, next_url) do - drop = - if cross_origin?(url, next_url), - do: @always_strip ++ @cross_origin_strip, - else: @always_strip - - Enum.reject(headers, fn {name, _value} -> String.downcase(to_string(name)) in drop end) - end - - defp cross_origin?(url, next_url), do: origin(url) != origin(next_url) - - defp origin(url) do - uri = URI.parse(url) - {uri.scheme, uri.host && String.downcase(uri.host), uri.port} - end - defp check_size(_path, nil), do: :ok defp check_size(path, max_size) do diff --git a/lib/bow/downloader.ex b/lib/bow/downloader.ex index 552b5af..0bfe5c6 100644 --- a/lib/bow/downloader.ex +++ b/lib/bow/downloader.ex @@ -20,23 +20,47 @@ defmodule Bow.Downloader do @impl true def get(url, path, opts) do - req_opts = [redirect: false, into: File.stream!(path), headers: Keyword.get(opts, :headers, [])] + req_opts = [ + url: url, + into: File.stream!(path), + headers: Keyword.get(opts, :headers, []), + max_redirects: Keyword.get(opts, :max_redirects, 5) + ] - case Req.get(url, req_opts) do - {:ok, %{status: status, headers: headers}} -> - {:ok, status, for({name, values} <- headers, value <- values, do: {name, value})} + case Req.run(req_opts) do + {req, %Req.Response{status: status} = resp} when status in 200..299 -> + {:ok, %{url: URI.to_string(req.url), headers: headers(resp)}} - {:error, reason} -> - {:error, reason} + {_req, %Req.Response{status: status} = resp} -> + {:error, %{status: status, headers: headers(resp)}} + + {_req, exception} -> + {:error, exception} end end + + defp headers(resp) do + for {name, values} <- resp.headers, value <- values, do: {name, value} + end end + + Req follows redirects, but on redirect to a different origin it removes only + the `authorization` header, other credentials like `cookie` are sent as they are. """ @type headers :: [{name :: String.t(), value :: String.t()}] + @typedoc """ + Successful response + + - `:url` - final URL after redirects, used for the file name. When missing, + the requested URL is used. + - `:headers` - response headers with lowercase names + """ + @type response :: %{required(:headers) => headers, optional(:url) => String.t()} + @doc """ - Make a single GET request + Download the file with a GET request Arguments: - `url` - URL to request @@ -47,12 +71,13 @@ defmodule Bow.Downloader do `{:error, :max_size_exceeded}` when it's exceeded - `:timeout` - total download time in milliseconds - Must not follow redirects, `Bow.Download` takes care of them and calls the downloader again - with the next URL. Options may differ between calls, e.g. credentials in `:headers` are removed - on redirect to a different origin, so always use the given `opts` instead of static configuration. + The downloader is responsible for following redirects (and supporting `:max_redirects` + if it makes sense for it). On redirect to a different origin (scheme, host or port) make sure + credentials like `authorization` and `cookie` from `:headers` do not leak to the new host. - Must return the response status and headers with lowercase names. + Returns `{:ok, response}` for 2xx responses and `{:error, %{status: status, headers: headers}}` + for other statuses. """ @callback get(url :: String.t(), path :: Path.t(), opts :: keyword) :: - {:ok, status :: pos_integer, headers} | {:error, any} + {:ok, response} | {:error, any} end diff --git a/lib/bow/downloader/httpc.ex b/lib/bow/downloader/httpc.ex index 50c8f17..d08ca10 100644 --- a/lib/bow/downloader/httpc.ex +++ b/lib/bow/downloader/httpc.ex @@ -7,11 +7,16 @@ defmodule Bow.Downloader.Httpc do The request runs in a separate process, so late `:httpc` messages (e.g. after a timeout) never reach the calling process. + Redirects are followed up to `:max_redirects` times. On redirect to a different origin + (scheme, host or port) credentials like `authorization` and `cookie` are removed from `:headers`. + Options: - `:timeout` - total download time in milliseconds (not the idle time between chunks), - defaults to `30_000`. Returns `{:error, :timeout}` when exceeded. + including redirects, defaults to `30_000`. Returns `{:error, :timeout}` when exceeded. - `:headers` - request headers as a list of `{name, value}` tuples, `user-agent` defaults to `"bow"` + - `:max_redirects` - maximum number of redirects to follow, defaults to `5`. + Returns `{:error, :too_many_redirects}` when exceeded. - `:max_size` - maximum body size in bytes. The download is aborted as soon as `content-length` or the received data exceeds it. Returns `{:error, :max_size_exceeded}`. """ @@ -20,13 +25,25 @@ defmodule Bow.Downloader.Httpc do @default_timeout 30_000 @default_headers [{"user-agent", "bow"}] + @max_redirects 5 + @redirect_statuses [301, 302, 303, 307, 308] + + # Headers filtering on redirect, per RFC 9110 section 15.4 (same lists as Tesla.Middleware.FollowRedirects). + # Hop-by-hop and conditional request headers do not carry over to a new request. + @always_strip ~w(connection keep-alive proxy-connection te trailer transfer-encoding upgrade + if-match if-modified-since if-none-match if-range if-unmodified-since) + # Credentials and origin bound headers must not leak to a different origin. + @cross_origin_strip ~w(authorization cookie host origin proxy-authorization referer) + # request/4 stops itself at the deadline, this only covers the cleanup @shutdown_margin 1_000 @impl true def get(url, path, opts) do timeout = Keyword.get(opts, :timeout, @default_timeout) - task = Task.async(fn -> request(url, path, opts, timeout) end) + max_redirects = Keyword.get(opts, :max_redirects, @max_redirects) + deadline = System.monotonic_time(:millisecond) + timeout + task = Task.async(fn -> follow(url, path, opts, deadline, max_redirects) end) case Task.yield(task, timeout + @shutdown_margin) || Task.shutdown(task, :brutal_kill) do {:ok, result} -> result @@ -35,13 +52,54 @@ defmodule Bow.Downloader.Httpc do end end - defp request(url, path, opts, timeout) do - state = %{ - path: path, - max_size: opts[:max_size], - deadline: System.monotonic_time(:millisecond) + timeout - } + defp follow(url, path, opts, deadline, redirects_left) do + case request(url, path, opts, deadline) do + {:ok, status, headers} when status in 200..299 -> + {:ok, %{url: url, headers: headers}} + + {:ok, status, _headers} when status in @redirect_statuses and redirects_left == 0 -> + {:error, :too_many_redirects} + + {:ok, status, headers} when status in @redirect_statuses -> + follow_redirect(url, path, opts, deadline, redirects_left, status, headers) + + {:ok, status, headers} -> + {:error, %{status: status, headers: headers}} + + {:error, reason} -> + {:error, reason} + end + end + + defp follow_redirect(url, path, opts, deadline, redirects_left, status, headers) do + with {_, location} <- List.keyfind(headers, "location", 0) do + next_url = url |> URI.merge(location) |> to_string() + opts = Keyword.update(opts, :headers, [], &filter_headers(&1, url, next_url)) + follow(next_url, path, opts, deadline, redirects_left - 1) + else + nil -> {:error, %{status: status, headers: headers}} + end + end + + defp filter_headers(headers, url, next_url) do + drop = + if cross_origin?(url, next_url), + do: @always_strip ++ @cross_origin_strip, + else: @always_strip + + Enum.reject(headers, fn {name, _value} -> String.downcase(to_string(name)) in drop end) + end + + defp cross_origin?(url, next_url), do: origin(url) != origin(next_url) + + defp origin(url) do + uri = URI.parse(url) + {uri.scheme, uri.host && String.downcase(uri.host), uri.port} + end + defp request(url, path, opts, deadline) do + state = %{path: path, max_size: opts[:max_size], deadline: deadline} + timeout = remaining(state) request = {String.to_charlist(url), request_headers(opts)} http_opts = [ @@ -118,7 +176,7 @@ defmodule Bow.Downloader.Httpc do end defp write_chunk(ref, file, size, chunk, state) do - with :ok <- IO.binwrite(file, chunk) do + with :ok <- :file.write(file, chunk) do stream_body(ref, file, size, state) else {:error, reason} -> cancel(ref, reason) diff --git a/test/bow/download_test.exs b/test/bow/download_test.exs index 6ac9e35..cd7fd6c 100644 --- a/test/bow/download_test.exs +++ b/test/bow/download_test.exs @@ -13,26 +13,12 @@ defmodule Bow.DownloadTest do assert File.read!(file.path) == File.read!(@file_cat) end - test "file with redirect" do + test "name from final URL returned by the downloader" do assert {:ok, file} = download("http://example.com/kitten.png") assert file.name == "cat.png" assert File.read!(file.path) == File.read!(@file_cat) end - test "file with relative redirect" do - assert {:ok, file} = download("http://example.com/relative/kitten.png") - assert file.name == "cat.png" - end - - test "too many redirects" do - assert {:error, :too_many_redirects} = download("http://example.com/loop.png") - end - - test "max redirects option" do - assert {:error, :too_many_redirects} = - download("http://example.com/kitten.png", max_redirects: 0) - end - test "file without content type" do assert {:ok, file} = download("http://example.com/notype.png") assert file.name == "notype.png" @@ -83,39 +69,15 @@ defmodule Bow.DownloadTest do assert {:ok, %Bow{name: "cat.png"}} = Bow.Download.download("http://example.com/cat.png") end - describe "headers on redirect" do - @headers [ - {"Authorization", "Bearer secret"}, - {"cookie", "session=1"}, - {"If-None-Match", "etag"}, - {"accept", "image/*"} - ] - - test "are passed to the downloader" do - assert {:ok, _} = download("http://example.com/cat.png", headers: @headers) - assert_received {:downloader_get, _url, _path, opts} - assert opts[:headers] == @headers - end - - test "keep credentials on the same origin" do - assert {:ok, _} = download("http://example.com/kitten.png", headers: @headers) - assert_received {:downloader_get, "http://example.com/kitten.png", _path, _opts} - assert_received {:downloader_get, "http://example.com/cat.png", _path, opts} - - assert opts[:headers] == [ - {"Authorization", "Bearer secret"}, - {"cookie", "session=1"}, - {"accept", "image/*"} - ] - end - - test "remove credentials on a different origin" do - assert {:ok, file} = download("http://example.com/to-cdn.png", headers: @headers) - assert file.name == "cat.png" - assert_received {:downloader_get, "http://example.com/to-cdn.png", _path, _opts} - assert_received {:downloader_get, "https://cdn.example.org/cat.png", _path, opts} - assert opts[:headers] == [{"accept", "image/*"}] - end + test "options are passed to the downloader" do + headers = [{"Authorization", "Bearer secret"}, {"accept", "image/*"}] + + assert {:ok, _} = + download("http://example.com/cat.png", headers: headers, max_redirects: 2) + + assert_received {:downloader_get, "http://example.com/cat.png", _path, opts} + assert opts[:headers] == headers + assert opts[:max_redirects] == 2 end test "max size is checked after download" do diff --git a/test/bow/downloader/httpc_test.exs b/test/bow/downloader/httpc_test.exs index 79c844e..236c253 100644 --- a/test/bow/downloader/httpc_test.exs +++ b/test/bow/downloader/httpc_test.exs @@ -25,9 +25,33 @@ defmodule Bow.Downloader.HttpcTest do end get "/redirect" do - conn - |> put_resp_header("location", "/cat.jpg") - |> send_resp(302, "") + redirect(conn, 302, "/cat.jpg") + end + + get "/relative/redirect" do + redirect(conn, 301, "../cat.jpg") + end + + get "/loop" do + redirect(conn, 307, "/loop") + end + + get "/slow-loop" do + Process.sleep(150) + redirect(conn, 302, "/slow-loop") + end + + get "/no-location" do + send_resp(conn, 302, "") + end + + get "/to-headers" do + redirect(conn, 302, "/headers") + end + + # same port, different host, so a different origin + get "/to-other-origin" do + redirect(conn, 302, "http://localhost:#{conn.port}/headers") end get "/slow" do @@ -60,6 +84,12 @@ defmodule Bow.Downloader.HttpcTest do match _ do send_resp(conn, 404, "nope") end + + defp redirect(conn, status, location) do + conn + |> put_resp_header("location", location) + |> send_resp(status, "") + end end setup_all do @@ -74,25 +104,78 @@ defmodule Bow.Downloader.HttpcTest do end test "writes body to file", %{base_url: base_url, path: path} do - assert {:ok, 200, headers} = Httpc.get(base_url <> "/cat.jpg", path, []) + assert {:ok, %{headers: headers}} = Httpc.get(base_url <> "/cat.jpg", path, []) assert {"content-type", "image/jpeg"} in headers assert File.read!(path) == File.read!(@file_cat) end test "streams big body to file", %{base_url: base_url, path: path} do - assert {:ok, 200, _headers} = Httpc.get(base_url <> "/big", path, []) + assert {:ok, _response} = Httpc.get(base_url <> "/big", path, []) assert File.stat!(path).size == @big_size end - test "does not follow redirects", %{base_url: base_url, path: path} do - assert {:ok, 302, headers} = Httpc.get(base_url <> "/redirect", path, []) - assert {"location", "/cat.jpg"} in headers + test "returns error with status of failed request", %{base_url: base_url, path: path} do + assert {:error, %{status: 404, headers: _}} = Httpc.get(base_url <> "/nope", path, []) assert File.read!(path) == "" end - test "returns status of failed request", %{base_url: base_url, path: path} do - assert {:ok, 404, _headers} = Httpc.get(base_url <> "/nope", path, []) - assert File.read!(path) == "" + describe "redirects" do + @headers [ + {"Authorization", "Bearer secret"}, + {"cookie", "session=1"}, + {"If-None-Match", "etag"}, + {"accept", "image/*"} + ] + + test "are followed", %{base_url: base_url, path: path} do + assert {:ok, %{url: url}} = Httpc.get(base_url <> "/redirect", path, []) + assert url == base_url <> "/cat.jpg" + assert File.read!(path) == File.read!(@file_cat) + end + + test "relative location", %{base_url: base_url, path: path} do + assert {:ok, %{url: url}} = Httpc.get(base_url <> "/relative/redirect", path, []) + assert url == base_url <> "/cat.jpg" + end + + test "too many redirects", %{base_url: base_url, path: path} do + assert {:error, :too_many_redirects} = Httpc.get(base_url <> "/loop", path, []) + end + + test "max redirects option", %{base_url: base_url, path: path} do + assert {:error, :too_many_redirects} = + Httpc.get(base_url <> "/redirect", path, max_redirects: 0) + end + + test "without location", %{base_url: base_url, path: path} do + assert {:error, %{status: 302}} = Httpc.get(base_url <> "/no-location", path, []) + end + + test "timeout covers all redirects", %{base_url: base_url, path: path} do + assert {:error, :timeout} = + Httpc.get(base_url <> "/slow-loop", path, timeout: 400, max_redirects: 10) + end + + test "keep credentials on the same origin", %{base_url: base_url, path: path} do + assert {:ok, _response} = Httpc.get(base_url <> "/to-headers", path, headers: @headers) + + body = File.read!(path) + assert body =~ "authorization: Bearer secret" + assert body =~ "cookie: session=1" + assert body =~ "accept: image/*" + refute body =~ "if-none-match" + end + + test "remove credentials on a different origin", %{base_url: base_url, path: path} do + assert {:ok, %{url: "http://localhost:" <> _}} = + Httpc.get(base_url <> "/to-other-origin", path, headers: @headers) + + body = File.read!(path) + refute body =~ "authorization" + refute body =~ "cookie" + refute body =~ "if-none-match" + assert body =~ "accept: image/*" + end end test "timeout", %{base_url: base_url, path: path} do @@ -112,13 +195,13 @@ defmodule Bow.Downloader.HttpcTest do end test "default user agent", %{base_url: base_url, path: path} do - assert {:ok, 200, _headers} = Httpc.get(base_url <> "/headers", path, []) + assert {:ok, _response} = Httpc.get(base_url <> "/headers", path, []) assert File.read!(path) =~ "user-agent: bow" end test "custom headers", %{base_url: base_url, path: path} do headers = [{"User-Agent", "my-app"}, {"x-token", "secret"}] - assert {:ok, 200, _headers} = Httpc.get(base_url <> "/headers", path, headers: headers) + assert {:ok, _response} = Httpc.get(base_url <> "/headers", path, headers: headers) body = File.read!(path) assert body =~ "user-agent: my-app" @@ -140,7 +223,7 @@ defmodule Bow.Downloader.HttpcTest do test "max size not exceeded", %{base_url: base_url, path: path} do size = File.stat!(@file_cat).size - assert {:ok, 200, _headers} = Httpc.get(base_url <> "/cat.jpg", path, max_size: size) + assert {:ok, _response} = Httpc.get(base_url <> "/cat.jpg", path, max_size: size) end test "connection error", %{path: path} do @@ -160,19 +243,27 @@ defmodule Bow.Downloader.HttpcTest do @impl true def get(url, path, opts) do req_opts = [ - redirect: false, + url: url, into: File.stream!(path), - headers: Keyword.get(opts, :headers, []) + headers: Keyword.get(opts, :headers, []), + max_redirects: Keyword.get(opts, :max_redirects, 5) ] - case Req.get(url, req_opts) do - {:ok, %{status: status, headers: headers}} -> - {:ok, status, for({name, values} <- headers, value <- values, do: {name, value})} + case Req.run(req_opts) do + {req, %Req.Response{status: status} = resp} when status in 200..299 -> + {:ok, %{url: URI.to_string(req.url), headers: headers(resp)}} + + {_req, %Req.Response{status: status} = resp} -> + {:error, %{status: status, headers: headers(resp)}} - {:error, reason} -> - {:error, reason} + {_req, exception} -> + {:error, exception} end end + + defp headers(resp) do + for {name, values} <- resp.headers, value <- values, do: {name, value} + end end test "Req downloader example", %{base_url: base_url} do diff --git a/test/support/test_downloader.ex b/test/support/test_downloader.ex index dcdd8ef..8f8cb26 100644 --- a/test/support/test_downloader.ex +++ b/test/support/test_downloader.ex @@ -11,11 +11,8 @@ defmodule Bow.TestDownloader do case url do "http://example.com/cat.png" -> ok(path, @file_cat, [{"content-type", "image/png"}]) - "http://example.com/kitten.png" -> redirect("http://example.com/cat.png") - "http://example.com/relative/kitten.png" -> redirect("../cat.png") - "http://example.com/loop.png" -> redirect("http://example.com/loop.png") - "http://example.com/to-cdn.png" -> redirect("https://cdn.example.org/cat.png") - "https://cdn.example.org/cat.png" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + # redirected to http://example.com/cat.png + "http://example.com/kitten.png" -> ok(path, @file_cat, "http://example.com/cat.png") "http://example.com/notype.png" -> ok(path, @file_cat, []) "http://example.com/noext" -> ok(path, @file_cat, []) "http://example.com/u" <> _ -> ok(path, @file_cat, [{"content-type", "image/png"}]) @@ -24,14 +21,18 @@ defmodule Bow.TestDownloader do "http://example.com/.weird-path" -> ok(path, @file_cat, [{"content-type", "image/png"}]) "http://example.com/bear.png" -> ok(path, @file_bear, [{"content-type", "image/png"}]) "http://example.com/broken" -> {:error, :econnrefused} - _ -> {:ok, 404, []} + _ -> {:error, %{status: 404, headers: []}} end end - defp ok(path, file, headers) do + defp ok(path, file, final_url) when is_binary(final_url) do File.cp!(file, path) - {:ok, 200, headers} + {:ok, %{url: final_url, headers: [{"content-type", "image/png"}]}} end - defp redirect(location), do: {:ok, 301, [{"location", location}]} + # without :url, like a downloader that does not know the final URL + defp ok(path, file, headers) do + File.cp!(file, path) + {:ok, %{headers: headers}} + end end