diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5915d96 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +jobs: + test: + name: Elixir ${{ matrix.elixir }} / OTP ${{ matrix.otp }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + # minimum supported + - elixir: "1.16" + otp: "26" + # latest + - elixir: "1.20" + otp: "28" + lint: true + env: + MIX_ENV: test + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: development + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + s3: + image: adobe/s3mock + env: + COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: test-bucket + ports: + - 4567:9090 + steps: + - uses: actions/checkout@v5 + + - uses: erlef/setup-beam@v1 + with: + elixir-version: ${{ matrix.elixir }} + otp-version: ${{ matrix.otp }} + + - uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ matrix.elixir }}-${{ matrix.otp }}-${{ hashFiles('mix.lock') }} + restore-keys: ${{ runner.os }}-mix-${{ matrix.elixir }}-${{ matrix.otp }}- + + - run: mix deps.get + + - if: matrix.lint + run: mix format --check-formatted + + - if: matrix.lint + run: mix deps.unlock --check-unused + + - if: matrix.lint + run: mix compile --warnings-as-errors + + - run: mix ecto.create + + - name: Wait for S3 mock + run: timeout 60 bash -c 'until curl -sf http://localhost:4567/ > /dev/null; do sleep 1; done' + + - run: mix test --include ecto --include s3 diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..a7baa92 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +erlang 28.5.0.2 +elixir 1.19.5-otp-28 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc93b6..280a033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +# v0.5.0 + +Breaking changes: + +* Minimum Elixir version is now 1.16 +* `Bow.Exec` and `erlexec` dependency removed. Generate versions in `transform/3` with + `Bow.with_output/2` and any tool you like (`System.cmd/3`, Vix, MuonTrap, erlexec...). + `:exec_timeout` config removed +* `tesla` dependency removed. Remote files are downloaded with `:httpc` by default (`Bow.Downloader.Httpc`), + use `config :bow, downloader: MyDownloader` to plug in your own `Bow.Downloader` (e.g. Req or Tesla). + Downloaders follow redirects themselves and return the final URL used for the file name +* `Bow.Download.download(client, url)` is now `Bow.Download.download(url, opts)` +* `Bow.Ecto.cast_uploads/4` and `Bow.Ecto.download_params/3` take options (e.g. `downloader: ...`) + instead of a Tesla client +* `Bow.Download.download/2` errors are `{:error, %{status: status, headers: headers}}`, + `{:error, :too_many_redirects}`, `{:error, :max_size_exceeded}` and `{:error, :timeout}` +* Default downloader sends `user-agent: bow`, override it with the `:headers` option + +Other changes: + +* Updated dependencies (Ecto 3.10+, Plug 1.14+, ExAws 2.4+) +* Add `Bow.with_output/2` helper for writing processed files +* Downloaded files are streamed to disk instead of being loaded into memory +* `Bow.Download.download/2` and `Bow.Ecto.cast_uploads/4` options: `:headers`, `:max_size` (file size limit) + and `:timeout` (total download time) +* `Bow.Downloader.Httpc` removes credentials (`authorization`, `cookie`, ...) from `:headers` on redirect + to a different origin, `:max_redirects` option (defaults to 5) +* Document SSRF risk of downloading user provided URLs and the ExAws 2.7 hackney `HEAD` issue when loading files from `Bow.Storage.S3` +* fix: `Bow.Download` no longer requires Ecto + # v0.4.3 * fix: handle valid URL without file path diff --git a/README.md b/README.md index edd33ff..0c4d2ac 100644 --- a/README.md +++ b/README.md @@ -14,18 +14,22 @@ File uploads for Elixir ```elixir def deps do [ - {:bow, "~> 0.4.3"}, + {:bow, "~> 0.5.0"}, # for AWS S3 support - {:ex_aws, "~> 2.0"}, - {:ex_aws_s3, "~> 2.0"}, - - # for Bow.Exec - {:erlexec, "~> 1.7.0"} + {:ex_aws, "~> 2.4"}, + {:ex_aws_s3, "~> 2.4"} ] end ``` +> **Note** +> Loading files with `Bow.Storage.S3` uses `ExAws.S3.download_file/4`, which sends a `HEAD` request. +> The hackney client of ExAws 2.7 crashes on `HEAD` responses, use the Req client instead: +> `config :ex_aws, http_client: ExAws.Request.Req` (requires `{:req, "~> 0.5"}`). + +Bow requires Elixir 1.16 or newer. + ## Usage ### Minimal uploader definition @@ -65,8 +69,12 @@ defmodule AttachmentUploader do # generate image thumbnail def transform(source, target, :thumb) do - Bow.Exec.exec source, target, - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) end @@ -186,6 +194,72 @@ defmodule AvatarUploader do end ``` +### Downloading remote files + +`Bow.Ecto.cast_uploads/4` downloads files given as `remote__url` params +(e.g. `remote_avatar_url`). Files are streamed to disk with Erlang `:httpc` by default. + +```elixir +changeset +|> Bow.Ecto.cast_uploads(params, [:avatar], + max_size: 10_000_000, + timeout: 15_000, + headers: [{"user-agent", "my-app"}] +) +``` + +The default downloader follows up to 5 redirects (`:max_redirects` option). On redirect to a different +origin, credentials like `authorization` and `cookie` are removed from `:headers`. + +> **Warning** +> The URL comes from users, so it can point to your internal services, e.g. the +> `http://169.254.169.254/` cloud metadata endpoint (SSRF). Validate the URL before downloading +> it and always set `:max_size` to protect the disk. + +You can use any HTTP client by implementing the `Bow.Downloader` behaviour +(see its docs for an example based on Req). Your downloader is then responsible for following +redirects without leaking credentials to a different origin: + +```elixir +# config/config.exs +config :bow, downloader: MyApp.BowDownloader +``` + +### Processing files + +`transform/3` can generate the version in any way you like. `Bow.with_output/2` helps with +the boring part: it gives you a temporary output path (with the target extension, so tools like +ImageMagick or libvips pick the right format), checks the file was written and sets it on the target. + +With [Vix](https://hexdocs.pm/vix): + +```elixir +def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + with {:ok, thumb} <- Vix.Vips.Operation.thumbnail(source.path, 250) do + Vix.Vips.Image.write_to_file(thumb, output_path) + end + end) +end +``` + +With an external program: + +```elixir +def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) +end +``` + +`System.cmd/3` does not support timeouts. Version generation is limited by `:version_timeout`, +but the OS process may keep running after it. If you need to kill it, use a library like +[MuonTrap](https://hexdocs.pm/muontrap) or [erlexec](https://hexdocs.pm/erlexec). + ### Using Bow in test environment It is best to use local storage adapter when testing. @@ -206,7 +280,8 @@ mix test #### Testing ecto integration ```bash -# edit config/config.exs +# start postgres, or use your own and set TEST_DB_USERNAME, TEST_DB_PASSWORD, TEST_DB_HOST, TEST_DB_PORT +docker run --rm -d -p 5432:5432 -e POSTGRES_USER=development -e POSTGRES_HOST_AUTH_METHOD=trust postgres:16-alpine # create test database MIX_ENV=test mix ecto.create @@ -218,11 +293,8 @@ mix test --only ecto #### Testing S3 adapter ```bash -# install fake-s3 gem -gem install fakes3 - -# start fake-s3 server -fakes3 -r tmp/s3 -p 4567 +# start S3 mock server +docker run --rm -d -p 4567:9090 -e COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS=test-bucket adobe/s3mock # run tests mix test --only s3 diff --git a/config/config.exs b/config/config.exs index 4e02310..71e0cb4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,22 +1,24 @@ -use Mix.Config +import Config if Mix.env() == :test do config :bow, Bow.Repo, username: System.get_env("TEST_DB_USERNAME") || "development", password: System.get_env("TEST_DB_PASSWORD") || "", database: "bow_ecto_test", - hostname: "localhost", + hostname: System.get_env("TEST_DB_HOST") || "localhost", + port: String.to_integer(System.get_env("TEST_DB_PORT") || "5432"), pool: Ecto.Adapters.SQL.Sandbox config :bow, ecto_repos: [Bow.Repo] - config :logger, level: :warn + config :logger, level: :warning # fakes3 configuration # https://github.com/jubos/fake-s3/wiki/Supported-Clients#elixir config :ex_aws, access_key_id: ["123", :instance_role], secret_access_key: ["asdf", :instance_role], - region: "fakes3" + region: "fakes3", + http_client: ExAws.Request.Req config :ex_aws, :s3, scheme: "http://", diff --git a/lib/bow.ex b/lib/bow.ex index f9dc433..8725188 100644 --- a/lib/bow.ex +++ b/lib/bow.ex @@ -9,7 +9,7 @@ defmodule Bow do storage_prefix: "priv/static/uploads", # storage directory prefix store_timeout: 30_000, # single version upload timeout - exec_timeout: 15_000, # single command execution timeout + version_timeout: 60_000, # single version processing timeout """ def storage, do: Application.get_env(:bow, :storage, Bow.Storage.Local) @@ -108,6 +108,70 @@ defmodule Bow do end end + @doc """ + Run `fun` with an output path for `target` and set the result on it + + Use it in `c:Bow.Uploader.transform/3` to write the processed file. + The output path has the same extension as `target` name, so tools choosing + the format based on extension (like libvips or ImageMagick) work out of the box. + The file is removed when the calling process exits. + + `fun` must write the file and return `:ok` or `{:error, reason}`. + + # thumbnail with Vix + def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + with {:ok, thumb} <- Vix.Vips.Operation.thumbnail(source.path, 250) do + Vix.Vips.Image.write_to_file(thumb, output_path) + end + end) + end + + # thumbnail with ImageMagick + def transform(source, target, :thumb) do + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path], stderr_to_stdout: true) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) + end + """ + @spec with_output(t, (output_path :: Path.t() -> :ok | {:error, any})) :: + {:ok, t} | {:error, any} + def with_output(target, fun) when is_function(fun, 1) do + tmp_path = Plug.Upload.random_file!("bow-output") + # random_file! does not support extensions, write output next to it and move it back + # so the file is still removed together with the process + output_path = tmp_path <> target.ext + + try do + case fun.(output_path) do + :ok -> + keep_output(target, output_path, tmp_path) + + {:error, reason} -> + {:error, reason} + + other -> + raise ArgumentError, + "expected :ok or {:error, reason} from with_output/2 function, got: #{inspect(other)}" + end + after + # no-op after a successful rename, removes partial output otherwise + File.rm(output_path) + end + end + + defp keep_output(target, output_path, tmp_path) do + if File.exists?(output_path) do + File.rename!(output_path, tmp_path) + {:ok, set(target, :path, tmp_path)} + else + {:error, :output_not_found} + end + end + defp make(up, f0, fx, versions, opts) when is_list(versions) do versions |> Enum.map(&Task.async(fn -> make(up, f0, fx, &1, opts) end)) diff --git a/lib/bow/download.ex b/lib/bow/download.ex index 168e9ad..bb87131 100644 --- a/lib/bow/download.ex +++ b/lib/bow/download.ex @@ -1,52 +1,64 @@ defmodule Bow.Download do - use Tesla, only: [:get] - - plug(Tesla.Middleware.FollowRedirects) - @doc """ Download file from given URL + + The file name is taken from the final URL returned by the downloader (after redirects), + or the requested URL, with extension based on the `content-type` header. + + > #### Untrusted URLs {: .warning} + > + > When the URL comes from users (e.g. `remote_avatar_url` params), it can point to + > internal services, like `http://169.254.169.254/` cloud metadata endpoint (SSRF). + > Validate the URL before downloading it and set `:max_size`. + + Options: + - `:downloader` - module implementing `Bow.Downloader`, defaults to `:downloader` config + or `Bow.Downloader.Httpc` + - `:headers` - request headers as a list of `{name, value}` tuples + - `:max_size` - maximum file size in bytes. The downloader should stop the download when it's + exceeded, Bow checks the downloaded file size as well. Returns `{:error, :max_size_exceeded}`. + - `:timeout` - download timeout in milliseconds, see the downloader docs for the default + + Other options (e.g. `:max_redirects`) are passed to the downloader. Redirects are followed + by the downloader, see `Bow.Downloader.Httpc` for the default behaviour. """ - @spec download(client :: Tesla.Client.t(), url :: String.t()) :: - {:ok, Bow.t()} | {:error, any} - def download(client \\ %Tesla.Client{}, url) do - case get!(client, encode(url)) do - %{status: 200, url: url, body: body} = env -> - base = - url - |> URI.parse() - |> case do - %{path: path} when not is_nil(path) -> path |> Path.basename() - _ -> "" - end + @spec download(url :: String.t(), opts :: keyword) :: {:ok, Bow.t()} | {:error, any} + def download(url, opts \\ []) do + {downloader, opts} = Keyword.pop(opts, :downloader, downloader()) + url = encode(url) + path = Plug.Upload.random_file!("bow-download") - name = - case Tesla.get_header(env, "content-type") do - nil -> - base + with {:ok, response} <- downloader.get(url, path, opts), + :ok <- check_size(path, opts[:max_size]) do + {:ok, Bow.new(name: name(response[:url] || url, response.headers), path: path)} + else + {:error, reason} -> + File.rm(path) + {:error, reason} + end + end - content_type -> - case MIME.extensions(content_type) do - [ext | _] -> rootname(base) <> "." <> ext - _ -> base - end - end + defp downloader, do: Application.get_env(:bow, :downloader, Bow.Downloader.Httpc) - path = Plug.Upload.random_file!("bow-download") + defp check_size(_path, nil), do: :ok - case File.write(path, body) do - :ok -> - {:ok, Bow.new(name: name, path: path)} + defp check_size(path, max_size) do + if File.stat!(path).size > max_size, do: {:error, :max_size_exceeded}, else: :ok + end - {:error, reason} -> - {:error, reason} - end + defp name(url, headers) do + base = + case URI.parse(url) do + %{path: path} when not is_nil(path) -> Path.basename(path) + _ -> "" + end - env -> - {:error, env} + with {_, content_type} <- List.keyfind(headers, "content-type", 0), + [ext | _] <- MIME.extensions(content_type) do + rootname(base) <> "." <> ext + else + _ -> base end - rescue - ex in Tesla.Error -> - {:error, ex} end # If path name is malformed, for example looks like this: ".some-data", @@ -57,11 +69,8 @@ defmodule Bow.Download do # name if we cannot read it properly. defp rootname(base) do case Path.rootname(base) do - "" -> - Ecto.UUID.generate() - - name -> - name + "" -> 16 |> :crypto.strong_rand_bytes() |> Base.encode16(case: :lower) + name -> name end end diff --git a/lib/bow/downloader.ex b/lib/bow/downloader.ex new file mode 100644 index 0000000..0bfe5c6 --- /dev/null +++ b/lib/bow/downloader.ex @@ -0,0 +1,83 @@ +defmodule Bow.Downloader do + @moduledoc """ + Behaviour for HTTP clients used by `Bow.Download` + + Bow uses `Bow.Downloader.Httpc` (based on Erlang `:httpc`) by default. + You can provide your own implementation, e.g. to use a different HTTP client, + a proxy or to mock requests in tests: + + config :bow, downloader: MyApp.BowDownloader + + The downloader can also be passed per call with the `:downloader` option + of `Bow.Download.download/2` and `Bow.Ecto.cast_uploads/4`. + + ## Example + + Implementation based on [Req](https://hexdocs.pm/req): + + defmodule MyApp.BowDownloader do + @behaviour Bow.Downloader + + @impl true + def get(url, path, opts) do + req_opts = [ + url: url, + into: File.stream!(path), + headers: Keyword.get(opts, :headers, []), + max_redirects: Keyword.get(opts, :max_redirects, 5) + ] + + case Req.run(req_opts) do + {req, %Req.Response{status: status} = resp} when status in 200..299 -> + {:ok, %{url: URI.to_string(req.url), headers: headers(resp)}} + + {_req, %Req.Response{status: status} = resp} -> + {:error, %{status: status, headers: headers(resp)}} + + {_req, exception} -> + {:error, exception} + end + end + + defp headers(resp) do + for {name, values} <- resp.headers, value <- values, do: {name, value} + end + end + + Req follows redirects, but on redirect to a different origin it removes only + the `authorization` header, other credentials like `cookie` are sent as they are. + """ + + @type headers :: [{name :: String.t(), value :: String.t()}] + + @typedoc """ + Successful response + + - `:url` - final URL after redirects, used for the file name. When missing, + the requested URL is used. + - `:headers` - response headers with lowercase names + """ + @type response :: %{required(:headers) => headers, optional(:url) => String.t()} + + @doc """ + Download the file with a GET request + + Arguments: + - `url` - URL to request + - `path` - path the response body must be written to when the status is 2xx + - `opts` - options given to `Bow.Download.download/2`, the downloader should support: + - `:headers` - request headers as a list of `{name, value}` tuples + - `:max_size` - maximum body size in bytes, stop the download and return + `{:error, :max_size_exceeded}` when it's exceeded + - `:timeout` - total download time in milliseconds + + The downloader is responsible for following redirects (and supporting `:max_redirects` + if it makes sense for it). On redirect to a different origin (scheme, host or port) make sure + credentials like `authorization` and `cookie` from `:headers` do not leak to the new host. + + Returns `{:ok, response}` for 2xx responses and `{:error, %{status: status, headers: headers}}` + for other statuses. + """ + @callback get(url :: String.t(), path :: Path.t(), opts :: keyword) :: + {:ok, response} | {:error, any} +end diff --git a/lib/bow/downloader/httpc.ex b/lib/bow/downloader/httpc.ex new file mode 100644 index 0000000..d08ca10 --- /dev/null +++ b/lib/bow/downloader/httpc.ex @@ -0,0 +1,233 @@ +defmodule Bow.Downloader.Httpc do + @moduledoc """ + Default `Bow.Downloader` based on Erlang `:httpc` + + The response body is streamed directly to the file. + HTTPS certificates are verified using the operating system CA store. + The request runs in a separate process, so late `:httpc` messages + (e.g. after a timeout) never reach the calling process. + + Redirects are followed up to `:max_redirects` times. On redirect to a different origin + (scheme, host or port) credentials like `authorization` and `cookie` are removed from `:headers`. + + Options: + - `:timeout` - total download time in milliseconds (not the idle time between chunks), + including redirects, defaults to `30_000`. Returns `{:error, :timeout}` when exceeded. + - `:headers` - request headers as a list of `{name, value}` tuples, + `user-agent` defaults to `"bow"` + - `:max_redirects` - maximum number of redirects to follow, defaults to `5`. + Returns `{:error, :too_many_redirects}` when exceeded. + - `:max_size` - maximum body size in bytes. The download is aborted as soon as + `content-length` or the received data exceeds it. Returns `{:error, :max_size_exceeded}`. + """ + + @behaviour Bow.Downloader + + @default_timeout 30_000 + @default_headers [{"user-agent", "bow"}] + @max_redirects 5 + @redirect_statuses [301, 302, 303, 307, 308] + + # Headers filtering on redirect, per RFC 9110 section 15.4 (same lists as Tesla.Middleware.FollowRedirects). + # Hop-by-hop and conditional request headers do not carry over to a new request. + @always_strip ~w(connection keep-alive proxy-connection te trailer transfer-encoding upgrade + if-match if-modified-since if-none-match if-range if-unmodified-since) + # Credentials and origin bound headers must not leak to a different origin. + @cross_origin_strip ~w(authorization cookie host origin proxy-authorization referer) + + # request/4 stops itself at the deadline, this only covers the cleanup + @shutdown_margin 1_000 + + @impl true + def get(url, path, opts) do + timeout = Keyword.get(opts, :timeout, @default_timeout) + max_redirects = Keyword.get(opts, :max_redirects, @max_redirects) + deadline = System.monotonic_time(:millisecond) + timeout + task = Task.async(fn -> follow(url, path, opts, deadline, max_redirects) end) + + case Task.yield(task, timeout + @shutdown_margin) || Task.shutdown(task, :brutal_kill) do + {:ok, result} -> result + {:exit, reason} -> {:error, reason} + nil -> {:error, :timeout} + end + end + + defp follow(url, path, opts, deadline, redirects_left) do + case request(url, path, opts, deadline) do + {:ok, status, headers} when status in 200..299 -> + {:ok, %{url: url, headers: headers}} + + {:ok, status, _headers} when status in @redirect_statuses and redirects_left == 0 -> + {:error, :too_many_redirects} + + {:ok, status, headers} when status in @redirect_statuses -> + follow_redirect(url, path, opts, deadline, redirects_left, status, headers) + + {:ok, status, headers} -> + {:error, %{status: status, headers: headers}} + + {:error, reason} -> + {:error, reason} + end + end + + defp follow_redirect(url, path, opts, deadline, redirects_left, status, headers) do + with {_, location} <- List.keyfind(headers, "location", 0) do + next_url = url |> URI.merge(location) |> to_string() + opts = Keyword.update(opts, :headers, [], &filter_headers(&1, url, next_url)) + follow(next_url, path, opts, deadline, redirects_left - 1) + else + nil -> {:error, %{status: status, headers: headers}} + end + end + + defp filter_headers(headers, url, next_url) do + drop = + if cross_origin?(url, next_url), + do: @always_strip ++ @cross_origin_strip, + else: @always_strip + + Enum.reject(headers, fn {name, _value} -> String.downcase(to_string(name)) in drop end) + end + + defp cross_origin?(url, next_url), do: origin(url) != origin(next_url) + + defp origin(url) do + uri = URI.parse(url) + {uri.scheme, uri.host && String.downcase(uri.host), uri.port} + end + + defp request(url, path, opts, deadline) do + state = %{path: path, max_size: opts[:max_size], deadline: deadline} + timeout = remaining(state) + request = {String.to_charlist(url), request_headers(opts)} + + http_opts = [ + timeout: timeout, + connect_timeout: timeout, + autoredirect: false, + ssl: ssl_opts() + ] + + case :httpc.request(:get, request, http_opts, + sync: false, + stream: :self, + body_format: :binary + ) do + {:ok, ref} -> receive_response(ref, state) + {:error, reason} -> {:error, reason} + end + end + + # :httpc streams only 200 and 206 responses, others are received at once + defp receive_response(ref, state) do + receive do + {:http, {^ref, :stream_start, headers}} -> + save_stream(ref, normalize(headers), state) + + {:http, {^ref, {{_version, status, _reason}, headers, body}}} -> + save_body(status, normalize(headers), body, state) + + {:http, {^ref, {:error, reason}}} -> + {:error, reason} + after + remaining(state) -> cancel(ref, :timeout) + end + end + + defp save_stream(ref, headers, state) do + with :ok <- check_content_length(headers, state.max_size), + {:ok, :ok} <- File.open(state.path, [:write, :binary], &stream_body(ref, &1, 0, state)) do + # Range header is never sent, so a streamed response can't be 206 + {:ok, 200, headers} + else + {:ok, {:error, reason}} -> {:error, reason} + {:error, reason} -> cancel(ref, reason) + end + end + + defp save_body(status, headers, body, state) when status in 200..299 do + with :ok <- check_size(byte_size(body), state.max_size), + :ok <- File.write(state.path, body) do + {:ok, status, headers} + end + end + + defp save_body(status, headers, _body, _state), do: {:ok, status, headers} + + defp stream_body(ref, file, size, state) do + receive do + {:http, {^ref, :stream, chunk}} -> + write_chunk(ref, file, size + byte_size(chunk), chunk, state) + + {:http, {^ref, :stream_end, _headers}} -> + :ok + + {:http, {^ref, {:error, reason}}} -> + {:error, reason} + after + remaining(state) -> cancel(ref, :timeout) + end + end + + defp write_chunk(ref, _file, size, _chunk, %{max_size: max_size}) + when is_integer(max_size) and size > max_size do + cancel(ref, :max_size_exceeded) + end + + defp write_chunk(ref, file, size, chunk, state) do + with :ok <- :file.write(file, chunk) do + stream_body(ref, file, size, state) + else + {:error, reason} -> cancel(ref, reason) + end + end + + defp check_content_length(headers, max_size) do + with {_, value} <- List.keyfind(headers, "content-length", 0), + {length, ""} <- Integer.parse(value) do + check_size(length, max_size) + else + _ -> :ok + end + end + + defp check_size(size, max_size) when is_integer(max_size) and size > max_size, + do: {:error, :max_size_exceeded} + + defp check_size(_size, _max_size), do: :ok + + defp remaining(state), do: max(state.deadline - System.monotonic_time(:millisecond), 0) + + defp cancel(ref, reason) do + :httpc.cancel_request(ref) + {:error, reason} + end + + defp request_headers(opts) do + headers = Keyword.get(opts, :headers, []) + names = MapSet.new(headers, fn {name, _value} -> String.downcase(to_string(name)) end) + defaults = Enum.reject(@default_headers, fn {name, _value} -> name in names end) + + for {name, value} <- defaults ++ headers do + {to_charlist(name), to_charlist(value)} + end + end + + defp normalize(headers) do + for {name, value} <- headers do + {name |> to_string() |> String.downcase(), to_string(value)} + end + end + + defp ssl_opts do + [ + verify: :verify_peer, + cacerts: :public_key.cacerts_get(), + depth: 4, + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ] + ] + end +end diff --git a/lib/bow/ecto.ex b/lib/bow/ecto.ex index 2aeb0f5..6d6b2cc 100644 --- a/lib/bow/ecto.ex +++ b/lib/bow/ecto.ex @@ -303,34 +303,33 @@ defmodule Bow.Ecto do Download remote files for given fields, i.e. `params["remote_avatar_url"] = "http://example.com/some/file.png"` + Options are passed to `Bow.Download.download/2`, e.g. `:max_size`, `:timeout` or `:headers`. + + > #### Untrusted URLs {: .warning} + > + > The URLs come from users, see `Bow.Download.download/2` about validating them + > and limiting the file size. + Example changeset |> cast(params, [:name, :avatar]) - |> Bow.Ecto.cast_uploads(params, [:avatar]) + |> Bow.Ecto.cast_uploads(params, [:avatar], max_size: 10_000_000) """ - @spec cast_uploads(any, map, list, Tesla.Client.t()) :: - Ecto.Changeset.t() - def cast_uploads(changeset, params, fields, client \\ %Tesla.Client{}) do - Ecto.Changeset.cast(changeset, download_params(params, fields, client), fields) + @spec cast_uploads(any, map, list, keyword) :: Ecto.Changeset.t() + def cast_uploads(changeset, params, fields, opts \\ []) do + Ecto.Changeset.cast(changeset, download_params(params, fields, opts), fields) end - @spec download_params(map, list, Tesla.Client.t()) :: map - def download_params(params, fields, client \\ %Tesla.Client{}) do + @spec download_params(map, list, keyword) :: map + def download_params(params, fields, opts \\ []) do Enum.reduce(fields, params, fn field, params -> field = to_string(field) - case params["remote_#{field}_url"] do - nil -> - params - - "" -> - params - - url -> - case Bow.Download.download(client, url) do - {:ok, file} -> Map.put(params, field, file) - _ -> params - end + with url when url not in [nil, ""] <- params["remote_#{field}_url"], + {:ok, file} <- Bow.Download.download(url, opts) do + Map.put(params, field, file) + else + _ -> params end end) end diff --git a/lib/bow/exec.ex b/lib/bow/exec.ex deleted file mode 100644 index ce0a6c3..0000000 --- a/lib/bow/exec.ex +++ /dev/null @@ -1,101 +0,0 @@ -defmodule Bow.Exec do - @moduledoc """ - Transform files with shell commands - - This module allows executing any external command taking care of temporary path generation and error handling. - It is as reliable as [erlexec](https://github.com/saleyn/erlexec) module (very!). - - It is also possible to provide custom command timeout. See `exec/4` to see all available options. - """ - - @type command :: [String.t() | {:input, integer} | :input | :output] - - defp default_timeout, do: Application.get_env(:bow, :exec_timeout, 15_000) - - @doc """ - Execute command - - Arguments: - - `source` - source file to be transformed - - `target_name` - target file - - `command` - the command to be executed. Placeholders `${input}` and `${output}` will be replaced with source and target paths - - Options: - - `:timeout` - time in which the command must return. If it's exceeded the command process will be killed. - - Examples - - # generate image thumbnail from first page of pdf - def transform(file, :pdf_thumbnail) do - Bow.Exec.exec file, filename(file, :pdf_thumbnail), - "convert '${input}[0]' -strip -gravity North -background '#ffffff'" <> - " -resize 250x175^ -extent 250x175 -format png png:${output}" - end - - """ - @spec exec(Bow.t(), Bow.t(), command, keyword) :: {:ok, Bow.t()} | {:error, any} - def exec(source, target, command, opts \\ []) do - timeout = opts[:timeout] || default_timeout() - - source_path = source.path - target_path = Plug.Upload.random_file!("bow-exec") - - cmd = - command - |> Enum.map(fn - {:input, idx} when is_integer(idx) -> "#{source_path}[#{idx}]" - :input -> source_path - :output -> target_path - {:output, format} -> "#{format}:#{target_path}" - arg -> arg - end) - |> Enum.map(&to_charlist/1) - - trapping(fn -> - case :exec.run_link(cmd, stdout: self(), stderr: self()) do - {:ok, pid, ospid} -> - case wait_for_exit(pid, ospid, timeout) do - {:ok, output} -> - if File.exists?(target_path) do - {:ok, Bow.set(target, :path, target_path)} - else - {:error, reason: :file_not_found, output: output, exit_code: 0, cmd: cmd} - end - - {:error, exit_code, output} -> - {:error, output: output, exit_code: exit_code, cmd: cmd} - end - - error -> - error - end - end) - end - - defp trapping(fun) do - trap = Process.flag(:trap_exit, true) - result = fun.() - Process.flag(:trap_exit, trap) - result - end - - defp wait_for_exit(pid, ospid, timout) do - receive do - {:EXIT, ^pid, :normal} -> {:ok, receive_output(ospid)} - {:EXIT, ^pid, {:exit_status, code}} -> {:error, code, receive_output(ospid)} - after - timout -> - :exec.stop_and_wait(pid, 2000) - {:error, :timeout, receive_output(ospid)} - end - end - - defp receive_output(ospid, output \\ []) do - receive do - {:stdout, ^ospid, data} -> receive_output(ospid, [output, data]) - {:stderr, ^ospid, data} -> receive_output(ospid, [output, data]) - after - 0 -> output |> to_string - end - end -end diff --git a/lib/bow/uploader.ex b/lib/bow/uploader.ex index b71b881..91c72d3 100644 --- a/lib/bow/uploader.ex +++ b/lib/bow/uploader.ex @@ -40,8 +40,12 @@ defmodule Bow.Uploader do # generate image thumbnail def transform(source, target, :thumb) do - Bow.Exec.exec source, target, - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) end @@ -178,11 +182,15 @@ defmodule Bow.Uploader do defmodule MyImageUploader do # generate image thumbnail def transform(source, target, :thumb) do - # Bow.Exec allows executing any system command replacing ${input} and ${output} - # with correct paths. It can also take :timeout option to prevent resource consumtion. - # Refer to Bow.Exec documentation for more details - Bow.Exec.exec source, target, - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" + # Bow.with_output gives a temporary output path (with target extension) + # and sets it on target when the file was written. + # Refer to Bow.with_output/2 documentation for more details + Bow.with_output(target, fn output_path -> + case System.cmd("convert", [source.path, "-resize", "250x175^", output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) end end @@ -191,8 +199,7 @@ defmodule Bow.Uploader do defmodule MyImageUploader do # generate image thumbnail and then micro_thumb version based on that def transform(source, target, :thumb) do - with {:ok, thumb_file} <- Bow.Exec.exec source, target - "convert ${input} -strip -gravity Center -resize 250x175^ -extent 250x175 ${output}" do + with {:ok, thumb_file} <- Bow.with_output(target, &make_thumb(source.path, &1)) do {:ok, thumb_file, [:micro_thumb]} end end diff --git a/mix.exs b/mix.exs index 740989b..67fb722 100644 --- a/mix.exs +++ b/mix.exs @@ -1,18 +1,17 @@ defmodule Bow.Mixfile do use Mix.Project - @version "0.4.3" + @version "0.5.0" def project do [ app: :bow, version: @version, - elixir: "~> 1.4", + elixir: "~> 1.16", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, aliases: aliases(), deps: deps(), - test_coverage: [tool: Coverex.Task], package: package(), dialyzer: dialyzer(), @@ -24,46 +23,30 @@ defmodule Bow.Mixfile do def application do [ - extra_applications: [:logger] ++ applications(Mix.env()) + extra_applications: [:logger, :crypto, :inets, :ssl, :public_key] ] end defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] - defp applications(:test) do - [ - # Bow.Ecto - :ecto, - :postgrex, - # Bow.Exec - :erlexec, - # Bow.Storage.S3 - :hackney, - :sweet_xml - ] - end - - defp applications(_), do: [] - defp deps do [ - {:plug, "~> 1.0"}, - {:tesla, "~> 1.0"}, - {:ecto, "~> 3.2", optional: true}, - {:ecto_sql, "~> 3.2", optional: true}, - {:erlexec, "~> 2.0", optional: true}, - {:ex_aws, "~> 2.0", optional: true}, - {:ex_aws_s3, "~> 2.0", optional: true}, + {:plug, "~> 1.14"}, + {:mime, "~> 1.0 or ~> 2.0"}, + {:ecto, "~> 3.10", optional: true}, + {:ecto_sql, "~> 3.10", optional: true}, + {:ex_aws, "~> 2.4", optional: true}, + {:ex_aws_s3, "~> 2.4", optional: true}, {:sweet_xml, "~> 0.7", optional: true}, # testing & docs {:jason, "~> 1.4", only: :test}, + {:req, "~> 0.5", only: :test}, + {:bandit, "~> 1.0", only: :test}, {:postgrex, ">= 0.0.0", only: :test}, - {:coverex, "~> 1.5", only: :test}, - {:ex_doc, "~> 0.21", only: :dev}, - {:mix_test_watch, "~> 1.1", only: :dev}, - {:dialyxir, "~> 1.1", only: :dev} + {:ex_doc, "~> 0.34", only: :dev, runtime: false}, + {:dialyxir, "~> 1.4", only: :dev, runtime: false} ] end @@ -96,7 +79,7 @@ defmodule Bow.Mixfile do defp dialyzer do [ - plt_add_apps: [:ecto, :ex_aws, :ex_aws_s3, :erlexec] + plt_add_apps: [:ecto, :ex_aws, :ex_aws_s3] ] end end diff --git a/mix.lock b/mix.lock index 5602677..1b2fa48 100644 --- a/mix.lock +++ b/mix.lock @@ -1,41 +1,32 @@ %{ - "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"}, - "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "coverex": {:hex, :coverex, "1.5.0", "a4248302f09562993041f1b056866bfd5688d3a03c429de80c47ea6663989ecc", [:mix], [{:hackney, "~> 1.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 3.1 or ~> 4.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "21a8f6e734a277b86c01b3cc44b7cc8b77cb1886adbebb708eefc6398567dcac"}, - "db_connection": {:hex, :db_connection, "2.4.1", "6411f6e23f1a8b68a82fa3a36366d4881f21f47fc79a9efb8c615e62050219da", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ea36d226ec5999781a9a8ad64e5d8c4454ecedc7a4d643e4832bf08efca01f00"}, - "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, - "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, - "earmark": {:hex, :earmark, "1.4.4", "4821b8d05cda507189d51f2caeef370cf1e18ca5d7dfb7d31e9cafe6688106a4", [:mix], [], "hexpm", "1f93aba7340574847c0f609da787f0d79efcab51b044bb6e242cae5aca9d264d"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.18", "e1b2be73eb08a49fb032a0208bf647380682374a725dfb5b9e510def8397f6f2", [:mix], [], "hexpm", "114a0e85ec3cf9e04b811009e73c206394ffecfcc313e0b346de0d557774ee97"}, - "ecto": {:hex, :ecto, "3.7.1", "a20598862351b29f80f285b21ec5297da1181c0442687f9b8329f0445d228892", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d36e5b39fc479e654cffd4dbe1865d9716e4a9b6311faff799b6f90ab81b8638"}, - "ecto_sql": {:hex, :ecto_sql, "3.7.1", "8de624ef50b2a8540252d8c60506379fbbc2707be1606853df371cf53df5d053", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.7.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2b42a32e2ce92f64aba5c88617891ab3b0ba34f3f3a503fa20009eae1a401c81"}, - "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "erlexec": {:hex, :erlexec, "2.2.2", "edb9f1a7d821a9df4efdc82bd11817ab5249dbd294d5c54ec249bd905f10e804", [:rebar3], [], "hexpm", "5e8e3c3773113785361b3b55218d92f7e91509cc9d679bf67c5c3703b394c900"}, - "ex_aws": {:hex, :ex_aws, "2.2.8", "8c142e516ab7c40e4eac1e2dbcda4c42ced48ff58ced81863fe4b7317c4de92b", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "84b51866078cc06970e608e4f57b4af246bb521ae650c284e82a7568bf436354"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.3.2", "92a63b72d763b488510626d528775b26831f5c82b066a63a3128054b7a09de28", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "b235b27131409bcc293c343bf39f1fbdd32892aa237b3f13752e914dc2979960"}, - "ex_doc": {:hex, :ex_doc, "0.26.0", "1922164bac0b18b02f84d6f69cab1b93bc3e870e2ad18d5dacb50a9e06b542a3", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2775d66e494a9a48355db7867478ffd997864c61c65a47d31c4949459281c78d"}, - "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm", "9a00246e8af58cdf465ae7c48fd6fd7ba2e43300413dfcc25447ecd3bf76f0c1"}, - "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mix_test_watch": {:hex, :mix_test_watch, "1.1.0", "330bb91c8ed271fe408c42d07e0773340a7938d8a0d281d57a14243eae9dc8c3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "52b6b1c476cbb70fd899ca5394506482f12e5f6b0d6acff9df95c7f1e0812ec3"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, - "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, - "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"}, - "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, - "poison": {:hex, :poison, "4.0.1", "bcb755a16fac91cad79bfe9fc3585bb07b9331e50cfe3420a24bcc2d735709ae", [:mix], [], "hexpm", "ba8836feea4b394bb718a161fc59a288fe0109b5006d6bdf97b6badfcf6f0f25"}, - "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.15.13", "7794e697481799aee8982688c261901de493eb64451feee6ea58207d7266d54a", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "3ffb76e1a97cfefe5c6a95632a27ffb67f28871c9741fb585f9d1c3cd2af70f1"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, - "sweet_xml": {:hex, :sweet_xml, "0.7.2", "4729f997286811fabdd8288f8474e0840a76573051062f066c4b597e76f14f9f", [:mix], [], "hexpm", "6894e68a120f454534d99045ea3325f7740ea71260bc315f82e29731d570a6e8"}, - "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, - "tesla": {:hex, :tesla, "1.4.3", "f5a494e08fb1abe4fd9c28abb17f3d9b62b8f6fc492860baa91efb1aab61c8a0", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "e0755bb664bf4d664af72931f320c97adbf89da4586670f4864bf259b5750386"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "bandit": {:hex, :bandit, "1.12.5", "af205a8e550f304caae09a97d29fd3c79a7f337526ea7cd772d2ff11d2f7c800", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.5", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "c5684ca062fa407cac115aec3256383f3e2ec9fdced7904d59cf5a7bb7ed6181"}, + "db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"}, + "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, + "dialyxir": {:hex, :dialyxir, "1.4.8", "7ef671a8aff9948b091d8c30f09467fbb16e77305cda451bce48109a0f5e021c", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "cbd5a851571e5dfeb32aaf2e840bfa98b7864cb3071bf2ef5d95d1276b12e072"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, + "ecto": {:hex, :ecto, "3.14.2", "99db28a864293a789c970651de711e3cae184291e0e7ea1166c54055ac41c1f3", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25d60b8c816a07d19d85b80bdf60978bd8b102209dda198d768cd7c6745339a6"}, + "ecto_sql": {:hex, :ecto_sql, "3.14.0", "06446ab8410d2f85bfbb80857ee224ab3b693700cbb38f6535d507449a627b2e", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.14.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.8", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f4d8d36faf294c9417b5a37ec7ac8217ee2abdef5fcf197ba690f361548d3949"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, + "ex_aws": {:hex, :ex_aws, "2.7.0", "e6bfd4b5fb8c791aa6c7d57fc7c45f050bb89de9576f1f6104aa974b234c01ec", [:mix], [{:configparser_ex, "~> 5.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 4.0", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bfe9d744d4fd4c1f40314ee7fab504d5547d1f01cd377fff1568cbe630b06d65"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.9", "862b7792f2e60d7010e2920d79964e3fab289bc0fd951b0ba8457a3f7f9d1199", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "a480d2bb2da64610014021629800e1e9457ca5e4a62f6775bffd963360c2bf90"}, + "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"}, + "finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"}, + "hpax": {:hex, :hpax, "1.1.0", "782931867cc23217c68fb5f68fe1a11f5e7544c7fda82c8a7019a5df5a4a1cdf", [:mix], [], "hexpm", "0b8d0f05832f55571d65ac720f79bf8994138ffbb133209dc4685eae0ad456a8"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mint": {:hex, :mint, "1.10.1", "c53e70867cf74017716884d8d33e0742b08b32e9cdb0031cbc69a429dc5555e3", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "0ba2a904605ed8406393444fb8b3356dc58eb59ee6c7fb94ac3f015e1be129e8"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "plug": {:hex, :plug, "1.20.3", "56c480c633ec2ce10140e236e15233bf576e1d323887d7c96711bd02ab5160db", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "be266aee1b8536ef6409d58cf39a3121319f0ec47cfa1b24024485aa0e76ad76"}, + "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, + "postgrex": {:hex, :postgrex, "0.22.4", "d271f595dfd25230b6398354e19d17bb5e2d20130fd2d9bdca7e15f125d43552", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "4aae45a2d60e35b04eea2602440be152fae332901f1fc7a60fc7cb7f0f9a9c5a"}, + "req": {:hex, :req, "0.7.4", "23e9ffec17de032a46a4b15ed65c09793893bf4a7c680f4bbf6227fce6bdf74d", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b192d63253e8dcc6221ef992ea9ebef7d3555166e8423aa5b553e86bc3c69a2"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, + "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, } diff --git a/test/bow/download_test.exs b/test/bow/download_test.exs index cba7c38..cd7fd6c 100644 --- a/test/bow/download_test.exs +++ b/test/bow/download_test.exs @@ -3,139 +3,92 @@ defmodule Bow.DownloadTest do @file_cat "test/files/cat.jpg" - import Bow.Download, only: [download: 2] - - setup do - client = - Tesla.client([Tesla.Middleware.FollowRedirects], fn env -> - case env do - %{url: "http://example.com/cat.png"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - %{url: "http://example.com/kitten.png"} -> - {:ok, %{env | status: 301, headers: [{"location", "http://example.com/cat.png"}]}} - - %{url: "http://example.com/loop.png"} -> - {:ok, %{env | status: 301, headers: [{"location", "http://example.com/loop.png"}]}} - - %{url: "http://example.com/notype.png"} -> - {:ok, %{env | status: 200, body: File.read!(@file_cat)}} - - %{url: "http://example.com/noext"} -> - {:ok, %{env | status: 200, body: File.read!(@file_cat)}} - - %{url: "http://example.com/u" <> _} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - %{url: "http://example.com"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - %{url: "http://example.com/dog.jpg"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "example/dog/nope"}] - }} - - %{url: "http://example.com/.weird-path"} -> - {:ok, - %{ - env - | status: 200, - body: File.read!(@file_cat), - headers: [{"content-type", "image/png"}] - }} - - _ -> - {:ok, %{env | status: 404, body: "NotFound"}} - end - end) - - {:ok, client: client} + defp download(url, opts \\ []) do + Bow.Download.download(url, [downloader: Bow.TestDownloader] ++ opts) end - test "regular file", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/cat.png") + test "regular file" do + assert {:ok, file} = download("http://example.com/cat.png") assert file.name == "cat.png" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file with redirect", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/kitten.png") + test "name from final URL returned by the downloader" do + assert {:ok, file} = download("http://example.com/kitten.png") assert file.name == "cat.png" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "too many redirects", %{client: client} do - assert {:error, %Tesla.Error{}} = download(client, "http://example.com/loop.png") - end - - test "file without content type", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/notype.png") + test "file without content type" do + assert {:ok, file} = download("http://example.com/notype.png") assert file.name == "notype.png" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file without extension", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/noext") + test "file without extension" do + assert {:ok, file} = download("http://example.com/noext") assert file.name == "noext" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file with invalid content type", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/dog.jpg") + test "file with invalid content type" do + assert {:ok, file} = download("http://example.com/dog.jpg") assert file.name == "dog.jpg" - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "url with path starting with dot", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/.weird-path") + test "url with path starting with dot" do + assert {:ok, file} = download("http://example.com/.weird-path") assert Regex.match?(~r/.+\.png/, file.name) - assert file.path != nil assert File.read!(file.path) == File.read!(@file_cat) end - test "file not found", %{client: client} do - assert {:error, %{status: 404}} = download(client, "http://example.com/nope") + test "file not found" do + assert {:error, %{status: 404}} = download("http://example.com/nope") end - test "without file path", %{client: client} do - assert {:ok, file} = download(client, "http://example.com") - - uuid_name = file.name |> String.split(".") |> Enum.at(0) + test "downloader error" do + assert {:error, :econnrefused} = download("http://example.com/broken") + end - assert {:ok, _} = Ecto.UUID.dump(uuid_name) - assert file.path != nil + test "without file path" do + assert {:ok, file} = download("http://example.com") + assert file.name =~ ~r/^[0-9a-f]{32}\.png$/ assert File.read!(file.path) == File.read!(@file_cat) end - test "dynamic URL", %{client: client} do - assert {:ok, file} = download(client, "http://example.com/u/91372?v=3&s=460") + test "dynamic URL" do + assert {:ok, file} = download("http://example.com/u/91372?v=3&s=460") assert file.name == "91372.png" end + + test "downloader from config" do + Application.put_env(:bow, :downloader, Bow.TestDownloader) + on_exit(fn -> Application.delete_env(:bow, :downloader) end) + + assert {:ok, %Bow{name: "cat.png"}} = Bow.Download.download("http://example.com/cat.png") + end + + test "options are passed to the downloader" do + headers = [{"Authorization", "Bearer secret"}, {"accept", "image/*"}] + + assert {:ok, _} = + download("http://example.com/cat.png", headers: headers, max_redirects: 2) + + assert_received {:downloader_get, "http://example.com/cat.png", _path, opts} + assert opts[:headers] == headers + assert opts[:max_redirects] == 2 + end + + test "max size is checked after download" do + assert {:error, :max_size_exceeded} = download("http://example.com/cat.png", max_size: 10) + assert_received {:downloader_get, _url, path, _opts} + refute File.exists?(path) + end + + test "removes file on error" do + assert {:error, %{status: 404}} = download("http://example.com/nope") + assert_received {:downloader_get, _url, path, _opts} + refute File.exists?(path) + end end diff --git a/test/bow/downloader/httpc_test.exs b/test/bow/downloader/httpc_test.exs new file mode 100644 index 0000000..236c253 --- /dev/null +++ b/test/bow/downloader/httpc_test.exs @@ -0,0 +1,274 @@ +defmodule Bow.Downloader.HttpcTest do + use ExUnit.Case + + alias Bow.Downloader.Httpc + + @file_cat "test/files/cat.jpg" + @big_size 3 * 1024 * 1024 + + defmodule Router do + use Plug.Router + + plug(:match) + plug(:dispatch) + + get "/cat.jpg" do + conn + |> put_resp_content_type("image/jpeg", nil) + |> send_file(200, "test/files/cat.jpg") + end + + get "/big" do + conn = send_chunked(conn, 200) + chunk = :binary.copy("a", 1024 * 1024) + Enum.reduce(1..3, conn, fn _, conn -> elem(chunk(conn, chunk), 1) end) + end + + get "/redirect" do + redirect(conn, 302, "/cat.jpg") + end + + get "/relative/redirect" do + redirect(conn, 301, "../cat.jpg") + end + + get "/loop" do + redirect(conn, 307, "/loop") + end + + get "/slow-loop" do + Process.sleep(150) + redirect(conn, 302, "/slow-loop") + end + + get "/no-location" do + send_resp(conn, 302, "") + end + + get "/to-headers" do + redirect(conn, 302, "/headers") + end + + # same port, different host, so a different origin + get "/to-other-origin" do + redirect(conn, 302, "http://localhost:#{conn.port}/headers") + end + + get "/slow" do + Process.sleep(1000) + send_resp(conn, 200, "late") + end + + # sends a chunk every 100ms for 1s, so it's never idle longer than the timeout + get "/drip" do + conn = send_chunked(conn, 200) + + # stop when the client disconnects after its timeout + Enum.reduce_while(1..10, conn, fn _, conn -> + Process.sleep(100) + + case chunk(conn, ".") do + {:ok, conn} -> {:cont, conn} + {:error, :closed} -> {:halt, conn} + end + end) + end + + get "/headers" do + body = + Enum.map_join(conn.req_headers, "\n", fn {name, value} -> "#{name}: #{value}" end) + + send_resp(conn, 200, body) + end + + match _ do + send_resp(conn, 404, "nope") + end + + defp redirect(conn, status, location) do + conn + |> put_resp_header("location", location) + |> send_resp(status, "") + end + end + + setup_all do + {:ok, server} = Bandit.start_link(plug: Router, ip: :loopback, port: 0, startup_log: false) + {:ok, {_ip, port}} = ThousandIsland.listener_info(server) + + %{base_url: "http://127.0.0.1:#{port}"} + end + + setup do + %{path: Plug.Upload.random_file!("httpc-test")} + end + + test "writes body to file", %{base_url: base_url, path: path} do + assert {:ok, %{headers: headers}} = Httpc.get(base_url <> "/cat.jpg", path, []) + assert {"content-type", "image/jpeg"} in headers + assert File.read!(path) == File.read!(@file_cat) + end + + test "streams big body to file", %{base_url: base_url, path: path} do + assert {:ok, _response} = Httpc.get(base_url <> "/big", path, []) + assert File.stat!(path).size == @big_size + end + + test "returns error with status of failed request", %{base_url: base_url, path: path} do + assert {:error, %{status: 404, headers: _}} = Httpc.get(base_url <> "/nope", path, []) + assert File.read!(path) == "" + end + + describe "redirects" do + @headers [ + {"Authorization", "Bearer secret"}, + {"cookie", "session=1"}, + {"If-None-Match", "etag"}, + {"accept", "image/*"} + ] + + test "are followed", %{base_url: base_url, path: path} do + assert {:ok, %{url: url}} = Httpc.get(base_url <> "/redirect", path, []) + assert url == base_url <> "/cat.jpg" + assert File.read!(path) == File.read!(@file_cat) + end + + test "relative location", %{base_url: base_url, path: path} do + assert {:ok, %{url: url}} = Httpc.get(base_url <> "/relative/redirect", path, []) + assert url == base_url <> "/cat.jpg" + end + + test "too many redirects", %{base_url: base_url, path: path} do + assert {:error, :too_many_redirects} = Httpc.get(base_url <> "/loop", path, []) + end + + test "max redirects option", %{base_url: base_url, path: path} do + assert {:error, :too_many_redirects} = + Httpc.get(base_url <> "/redirect", path, max_redirects: 0) + end + + test "without location", %{base_url: base_url, path: path} do + assert {:error, %{status: 302}} = Httpc.get(base_url <> "/no-location", path, []) + end + + test "timeout covers all redirects", %{base_url: base_url, path: path} do + assert {:error, :timeout} = + Httpc.get(base_url <> "/slow-loop", path, timeout: 400, max_redirects: 10) + end + + test "keep credentials on the same origin", %{base_url: base_url, path: path} do + assert {:ok, _response} = Httpc.get(base_url <> "/to-headers", path, headers: @headers) + + body = File.read!(path) + assert body =~ "authorization: Bearer secret" + assert body =~ "cookie: session=1" + assert body =~ "accept: image/*" + refute body =~ "if-none-match" + end + + test "remove credentials on a different origin", %{base_url: base_url, path: path} do + assert {:ok, %{url: "http://localhost:" <> _}} = + Httpc.get(base_url <> "/to-other-origin", path, headers: @headers) + + body = File.read!(path) + refute body =~ "authorization" + refute body =~ "cookie" + refute body =~ "if-none-match" + assert body =~ "accept: image/*" + end + end + + test "timeout", %{base_url: base_url, path: path} do + assert {:error, :timeout} = Httpc.get(base_url <> "/slow", path, timeout: 100) + end + + test "timeout is the total download time", %{base_url: base_url, path: path} do + started_at = System.monotonic_time(:millisecond) + assert {:error, :timeout} = Httpc.get(base_url <> "/drip", path, timeout: 300) + assert System.monotonic_time(:millisecond) - started_at < 800 + end + + test "late messages do not reach the caller", %{base_url: base_url, path: path} do + assert {:error, :timeout} = Httpc.get(base_url <> "/drip", path, timeout: 150) + assert {:error, :timeout} = Httpc.get(base_url <> "/slow", path, timeout: 100) + refute_receive {:http, _}, 1_500 + end + + test "default user agent", %{base_url: base_url, path: path} do + assert {:ok, _response} = Httpc.get(base_url <> "/headers", path, []) + assert File.read!(path) =~ "user-agent: bow" + end + + test "custom headers", %{base_url: base_url, path: path} do + headers = [{"User-Agent", "my-app"}, {"x-token", "secret"}] + assert {:ok, _response} = Httpc.get(base_url <> "/headers", path, headers: headers) + + body = File.read!(path) + assert body =~ "user-agent: my-app" + refute body =~ "user-agent: bow" + assert body =~ "x-token: secret" + end + + test "max size exceeded by content-length", %{base_url: base_url, path: path} do + assert {:error, :max_size_exceeded} = Httpc.get(base_url <> "/cat.jpg", path, max_size: 100) + assert File.read!(path) == "" + end + + test "max size exceeded while streaming", %{base_url: base_url, path: path} do + assert {:error, :max_size_exceeded} = + Httpc.get(base_url <> "/big", path, max_size: 1024 * 1024) + + assert File.stat!(path).size <= 1024 * 1024 + end + + test "max size not exceeded", %{base_url: base_url, path: path} do + size = File.stat!(@file_cat).size + assert {:ok, _response} = Httpc.get(base_url <> "/cat.jpg", path, max_size: size) + end + + test "connection error", %{path: path} do + assert {:error, {:failed_connect, _}} = Httpc.get("http://127.0.0.1:1/cat.jpg", path, []) + end + + test "default downloader of Bow.Download", %{base_url: base_url} do + assert {:ok, file} = Bow.Download.download(base_url <> "/redirect") + assert file.name == "cat.jpg" + assert File.read!(file.path) == File.read!(@file_cat) + end + + defmodule ReqDownloader do + # example from Bow.Downloader docs + @behaviour Bow.Downloader + + @impl true + def get(url, path, opts) do + req_opts = [ + url: url, + into: File.stream!(path), + headers: Keyword.get(opts, :headers, []), + max_redirects: Keyword.get(opts, :max_redirects, 5) + ] + + case Req.run(req_opts) do + {req, %Req.Response{status: status} = resp} when status in 200..299 -> + {:ok, %{url: URI.to_string(req.url), headers: headers(resp)}} + + {_req, %Req.Response{status: status} = resp} -> + {:error, %{status: status, headers: headers(resp)}} + + {_req, exception} -> + {:error, exception} + end + end + + defp headers(resp) do + for {name, values} <- resp.headers, value <- values, do: {name, value} + end + end + + test "Req downloader example", %{base_url: base_url} do + assert {:ok, file} = Bow.Download.download(base_url <> "/redirect", downloader: ReqDownloader) + assert file.name == "cat.jpg" + assert File.read!(file.path) == File.read!(@file_cat) + end +end diff --git a/test/bow/ecto_test.exs b/test/bow/ecto_test.exs index 9d6d313..bd19880 100644 --- a/test/bow/ecto_test.exs +++ b/test/bow/ecto_test.exs @@ -363,58 +363,35 @@ defmodule Bow.EctoTest do end describe "Remote file URLs" do - defp client do - Tesla.client([], fn - %{url: "http://example.com/bear.png"} = env -> - {:ok, - %{ - env - | status: 200, - body: File.read!("test/files/bear.png"), - headers: [{"Content-Type", "image/png"}] - }} - - %{url: "http://example.com"} = env -> - {:ok, - %{ - env - | status: 200, - body: File.read!("test/files/bear.png"), - headers: [{"Content-Type", "image/png"}] - }} - - env -> - {:ok, %{env | status: 404}} - end) - end + @opts [downloader: Bow.TestDownloader] test "empty params" do params = %{} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert user.changes == %{} end test "empty string as param" do params = %{"remote_avatar_url" => ""} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert user.changes == %{} end test "invalid URL" do params = %{"remote_avatar_url" => "some-ribbish"} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert user.changes == %{} end test "valid URL" do params = %{"remote_avatar_url" => "http://example.com/bear.png"} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert %Bow{} = user.changes.avatar end test "valid URL without file path" do params = %{"remote_avatar_url" => "http://example.com"} - user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], client()) + user = %User{} |> Bow.Ecto.cast_uploads(params, [:avatar], @opts) assert %Bow{} = user.changes.avatar end end diff --git a/test/bow/exec_test.exs b/test/bow/exec_test.exs deleted file mode 100644 index d5ab7dc..0000000 --- a/test/bow/exec_test.exs +++ /dev/null @@ -1,52 +0,0 @@ -defmodule Bow.ExecTest do - use ExUnit.Case - - @file_cat "test/files/cat.jpg" - - import Bow.Exec, only: [exec: 3, exec: 4] - - setup do - source = Bow.new(path: @file_cat) - target = Bow.set(source, :name, "thumb_#{source.name}") - - {:ok, source: source, target: target} - end - - test "successful command", %{source: source, target: target} do - assert {:ok, %Bow{path: path}} = - exec(source, target, ["test/scripts/copy.sh", :input, :output]) - - assert path != nil - assert File.exists?(path) - end - - test "command not found", %{source: source, target: target} do - assert {:error, reason} = exec(source, target, ["test/scripts/notfound", :input, :output]) - assert reason[:output] =~ ~r/No such file/u - end - - test "failing command", %{source: source, target: target} do - assert {:error, reason} = exec(source, target, ["test/scripts/fail.sh", :input, :output]) - assert reason[:exit_code] != 0 - end - - test "timout", %{source: source, target: target} do - assert {:error, reason} = - exec(source, target, ["test/scripts/sleep.sh", :input, :output], timeout: 500) - - assert reason[:exit_code] == :timeout - end - - test "file name with quotes" do - file = "tmp/asdf'weird$name.pdf" - File.write!(file, "data") - source = Bow.new(path: file) - target = Bow.set(source, :name, "thumb_#{source.name}") - - assert {:ok, %Bow{path: path}} = - exec(source, target, ["test/scripts/copy.sh", :input, :output]) - - assert path != nil - assert File.exists?(path) - end -end diff --git a/test/bow/with_output_test.exs b/test/bow/with_output_test.exs new file mode 100644 index 0000000..5ec9a45 --- /dev/null +++ b/test/bow/with_output_test.exs @@ -0,0 +1,93 @@ +defmodule Bow.WithOutputTest do + use ExUnit.Case + + @file_cat "test/files/cat.jpg" + + setup do + File.write!("tmp/text.txt", "hello") + source = Bow.new(path: "tmp/text.txt") + target = Bow.set(source, :name, "upcase.md") + + {:ok, source: source, target: target} + end + + test "sets written file path on target", %{source: source, target: target} do + assert {:ok, %Bow{name: "upcase.md", path: path}} = + Bow.with_output(target, fn output_path -> + send(self(), {:output_path, output_path}) + File.write(output_path, source.path |> File.read!() |> String.upcase()) + end) + + assert File.read!(path) == "HELLO" + assert_received {:output_path, output_path} + assert Path.extname(output_path) == ".md" + refute File.exists?(output_path) + end + + test "returns error and removes partial output", %{target: target} do + assert {:error, :boom} = + Bow.with_output(target, fn output_path -> + File.write!(output_path, "partial") + send(self(), {:output_path, output_path}) + {:error, :boom} + end) + + assert_received {:output_path, output_path} + refute File.exists?(output_path) + end + + test "removes partial output when function raises", %{target: target} do + assert_raise RuntimeError, "boom", fn -> + Bow.with_output(target, fn output_path -> + File.write!(output_path, "partial") + send(self(), {:output_path, output_path}) + raise "boom" + end) + end + + assert_received {:output_path, output_path} + refute File.exists?(output_path) + end + + test "output not written", %{target: target} do + assert {:error, :output_not_found} = Bow.with_output(target, fn _output_path -> :ok end) + end + + test "invalid return value", %{target: target} do + assert_raise ArgumentError, ~r/expected :ok or {:error, reason}.*got: {"", 0}/, fn -> + Bow.with_output(target, fn _output_path -> {"", 0} end) + end + end + + test "external command", %{source: source, target: target} do + assert {:ok, %Bow{path: path}} = + Bow.with_output(target, fn output_path -> + case System.cmd("cp", [source.path, output_path]) do + {_, 0} -> :ok + {cmd_output, exit_code} -> {:error, exit_code: exit_code, output: cmd_output} + end + end) + + assert File.read!(path) == "hello" + end + + test "works in uploader transform" do + defmodule ThumbUploader do + use Bow.Uploader + + def versions(_), do: [:original, :thumb] + + def transform(source, target, :thumb) do + Bow.with_output(target, &File.cp(source.path, &1)) + end + + def transform(source, target, version), do: super(source, target, version) + + def store_dir(_), do: "with_output" + end + + file = ThumbUploader.new(path: @file_cat) + assert {:ok, _} = Bow.store(file) + assert File.read!("tmp/bow/with_output/thumb_cat.jpg") == File.read!(@file_cat) + end +end diff --git a/test/bow_test.exs b/test/bow_test.exs index bc9452e..cc5d91d 100644 --- a/test/bow_test.exs +++ b/test/bow_test.exs @@ -424,7 +424,7 @@ defmodule BowTest do def versions(_), do: [:original, :pdf] def transform(source, target, :original), do: transform_original(source, target) - def transform(_source, _target, :pdf), do: %{}.foo + 1 + def transform(_source, _target, :pdf), do: raise(KeyError, key: :foo, term: %{}) def store_dir(_), do: "raise" end diff --git a/test/scripts/copy.sh b/test/scripts/copy.sh deleted file mode 100755 index 2f2e8ab..0000000 --- a/test/scripts/copy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -e -cp "$1" "$2" diff --git a/test/scripts/fail.sh b/test/scripts/fail.sh deleted file mode 100755 index b9e71c6..0000000 --- a/test/scripts/fail.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exit 5 diff --git a/test/scripts/sleep.sh b/test/scripts/sleep.sh deleted file mode 100755 index 9eaac3d..0000000 --- a/test/scripts/sleep.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -sleep 10000000 diff --git a/test/support/test_downloader.ex b/test/support/test_downloader.ex new file mode 100644 index 0000000..8f8cb26 --- /dev/null +++ b/test/support/test_downloader.ex @@ -0,0 +1,38 @@ +defmodule Bow.TestDownloader do + @moduledoc false + @behaviour Bow.Downloader + + @file_cat "test/files/cat.jpg" + @file_bear "test/files/bear.png" + + @impl true + def get(url, path, opts) do + send(self(), {:downloader_get, url, path, opts}) + + case url do + "http://example.com/cat.png" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + # redirected to http://example.com/cat.png + "http://example.com/kitten.png" -> ok(path, @file_cat, "http://example.com/cat.png") + "http://example.com/notype.png" -> ok(path, @file_cat, []) + "http://example.com/noext" -> ok(path, @file_cat, []) + "http://example.com/u" <> _ -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com/dog.jpg" -> ok(path, @file_cat, [{"content-type", "example/dog/nope"}]) + "http://example.com/.weird-path" -> ok(path, @file_cat, [{"content-type", "image/png"}]) + "http://example.com/bear.png" -> ok(path, @file_bear, [{"content-type", "image/png"}]) + "http://example.com/broken" -> {:error, :econnrefused} + _ -> {:error, %{status: 404, headers: []}} + end + end + + defp ok(path, file, final_url) when is_binary(final_url) do + File.cp!(file, path) + {:ok, %{url: final_url, headers: [{"content-type", "image/png"}]}} + end + + # without :url, like a downloader that does not know the final URL + defp ok(path, file, headers) do + File.cp!(file, path) + {:ok, %{headers: headers}} + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 788f6ec..ab0f4ce 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,3 @@ {:ok, _} = Application.ensure_all_started(:ecto_sql) +File.mkdir_p!("tmp") ExUnit.start(exclude: [:s3, :ecto])