diff --git a/.gitignore b/.gitignore index 9c8a180a2..396dfc981 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /_build /deps /tmp +/src/safe_erl_term.erl erl_crash.dump *.ez diff --git a/.travis.yml b/.travis.yml index 211da7e34..231be3961 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,63 +2,68 @@ language: erlang otp_release: 19.0 sudo: false addons: - postgresql: "9.4" + postgresql: '9.4' before_install: - - wget https://s3.amazonaws.com/travis-otp-releases/binaries/ubuntu/12.04/x86_64/erlang-${HEXWEB_OTP}-nonroot.tar.bz2 - - mkdir -p ${HEXWEB_OTP_PATH} - - tar -xf erlang-${HEXWEB_OTP}-nonroot.tar.bz2 -C ${HEXWEB_OTP_PATH} --strip-components=1 - - ${HEXWEB_OTP_PATH}/Install -minimal $(pwd)/${HEXWEB_OTP_PATH} - - - wget http://s3.hex.pm/builds/elixir/${HEXWEB_ELIXIR}.zip - - unzip -d ${HEXWEB_ELIXIR_PATH} ${HEXWEB_ELIXIR}.zip - - - wget http://s3.hex.pm/builds/elixir/${ELIXIR}.zip - - unzip -d elixir ${ELIXIR}.zip - - export PATH=$(pwd)/elixir/bin:${PATH} - - - mkdir -p ${HEXWEB_MIX_HOME} - - PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.hex --force - - PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.rebar --force - - mix local.hex --force - - mix local.rebar --force +- wget https://s3.amazonaws.com/travis-otp-releases/binaries/ubuntu/12.04/x86_64/erlang-${HEXWEB_OTP}-nonroot.tar.bz2 +- mkdir -p ${HEXWEB_OTP_PATH} +- tar -xf erlang-${HEXWEB_OTP}-nonroot.tar.bz2 -C ${HEXWEB_OTP_PATH} --strip-components=1 +- "${HEXWEB_OTP_PATH}/Install -minimal $(pwd)/${HEXWEB_OTP_PATH}" +- wget http://s3.hex.pm/builds/elixir/${HEXWEB_ELIXIR}.zip +- unzip -d ${HEXWEB_ELIXIR_PATH} ${HEXWEB_ELIXIR}.zip +- wget http://s3.hex.pm/builds/elixir/${ELIXIR}.zip +- unzip -d elixir ${ELIXIR}.zip +- export PATH=$(pwd)/elixir/bin:${PATH} +- mkdir -p ${HEXWEB_MIX_HOME} +- PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} + MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.hex --force +- PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} + MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.rebar --force +- mix local.hex --force +- mix local.rebar --force before_script: - - git clone https://github.com/hexpm/hex_web.git - - cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix deps.get; cd .. - - cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix compile; cd .. - - mix deps.get - - MIX_ENV=test mix deps.compile +- git clone https://github.com/hexpm/hex_web.git +- cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} + MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} + MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix deps.get; cd .. +- cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} + MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} + MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix compile; cd .. +- mix deps.get +- MIX_ENV=test mix deps.compile script: - - MIX_ENV=test mix compile - - mix test +- MIX_ENV=test mix compile +- mix test env: global: - - HEXWEB_OTP=19.0 - - HEXWEB_ELIXIR=v1.3.1 - - HEXWEB_PATH=hex_web - - HEXWEB_ELIXIR_PATH=hexweb_elixir - - HEXWEB_OTP_PATH=hexweb_otp - - HEXWEB_MIX_HOME=hexweb_mix - - HEXWEB_MIX_ARCHIVES=hexweb_mix + - HEXWEB_OTP=19.0 + - HEXWEB_ELIXIR=v1.3.1 + - HEXWEB_PATH=hex_web + - HEXWEB_ELIXIR_PATH=hexweb_elixir + - HEXWEB_OTP_PATH=hexweb_otp + - HEXWEB_MIX_HOME=hexweb_mix + - HEXWEB_MIX_ARCHIVES=hexweb_mix matrix: - - ELIXIR=v1.2.6 - - ELIXIR=v1.3.2 - - ELIXIR=master + - ELIXIR=v1.2.6 + - ELIXIR=v1.3.2 + - ELIXIR=master matrix: include: - - otp_release: 18.3 - env: ELIXIR=v1.0.5 - - otp_release: 18.3 - env: ELIXIR=v1.1.1 - - otp_release: 18.3 - env: ELIXIR=v1.2.6 - - otp_release: 18.3 - env: ELIXIR=v1.3.2 - - otp_release: 18.3 - env: ELIXIR=master - - otp_release: 17.5 - env: ELIXIR=v1.0.5 - - otp_release: 17.5 - env: ELIXIR=v1.1.1 + - otp_release: 18.3 + env: ELIXIR=v1.0.5 + - otp_release: 18.3 + env: ELIXIR=v1.1.1 + - otp_release: 18.3 + env: ELIXIR=v1.2.6 + - otp_release: 18.3 + env: ELIXIR=v1.3.2 + - otp_release: 18.3 + env: ELIXIR=master + - otp_release: 17.5 + env: ELIXIR=v1.0.5 + - otp_release: 17.5 + env: ELIXIR=v1.1.1 notifications: recipients: - - eric.meadows.jonsson@gmail.com + - eric.meadows.jonsson@gmail.com + slack: + secure: T5QN9luR7rMH4v88yD944QW3C3Itgd3Bh89oJbxNDweC2dyTqOuvJOgzfcFwBtnaT5j/kHOobd4QL7VHFZ+0UawOxxuy8fl7+A9mn02xVOIpn3Ky0qbpW//oApwKVz2x3X3OhuJASYDNj/w074WrjycIuk2iYkjCpIAnY2dSjo0= diff --git a/CHANGELOG.md b/CHANGELOG.md index c4de622bb..2343388dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,73 @@ -## v0.13.1-dev +## v0.15.1-dev + +## v0.15.0 (2016-12-24) + +### Package retirement + +With this new release you can mark versions of your packages as retired when you +no longer recommend its use. This can be because the release has a serious +security flaw, something went wrong with the release so that it's unusable or because +the package has been renamed or deprecated. A retired version is still usable +and fetchable but it will show as retired on hex.pm and when resolved Hex will +show a warning to the user with the retirement message. + +### Enhancements + * Add --module flag to `hex.docs` task + * Changed `hex.outdated` task to show if a dependency can be updated + * Add `hex.retire` task for package retirement + * Warn when resolving retired packages + * Restrict number of default SSL ciphers + +### Bug fixes + * Do not make conditional HTTP request if file is missing + * Ensure cache file is saved when Hex exits + +## v0.14.1 (2016-11-24) + +### Enhancements + * Add environment variable `HEX_HTTP_CONCURRENCY` for limiting number of concurrent HTTP requests + +### Bug fixes + * Fix compatibilities with older Elixir version (<= 1.1) + * Ensure build tools are unique in mix.lock and when publishing + * Fix `hex.docs open` opening websites on Unix systems + * Do not crash on diverged dependencies with conflicting SCMs + * Fix some duplicate HTTP requests on slow networks + * Limit concurrent registry HTTP requests + +## v0.14.0 (2016-10-28) + +### New registry format + +Hex has switched to a new registry format that is more efficient and will scale better as the registry grows. The new registry format is encoded with protocol buffers and is split into multiple files (one file per package) to avoid fetching one big file with data you will not need. The resolver will make more HTTP requests but will in total fetch much less data. The specification for the new format can be found here: https://github.com/hexpm/specifications/pull/10. The old ETS based registry format is no longer supported in the client but will continue to be available from the registry for the foreseeable future. + +### Enhancements + * `hex.docs open` will by default open the online hexdocs for the given package + * An `--offline` option has been added to `hex.docs open` for opening docs stored on your local filesystem and it will automatically fetch the docs if they are not available locally + * Only support secure SSL ciphers and safe SSL versions (support for SSLv3 has been dropped) + * Improvements to the language in the resolver error messages + +### Bug fixes + * Fix an issue where duplicate build tool names could be added to the package metadata + +## v0.13.2 (2016-09-19) + +### Bug fixes + * Only error on non-Hex dependencies when building + +## v0.13.1 (2016-09-19) + +### Enhancements + * Most warnings on `hex.publish` are now errors + +### Bug fixes + * Fix bug where the old config format was not readable + * Convert old config format to new format on every read + * Fix `HEX_UNSAFE_REGISTRY` negation ## v0.13.0 (2016-07-30) -* Enhancements +### Enhancements * Inform about new Hex version in `hex.info` * Support `extra` metadata field * Print package checksum when building and publishing @@ -18,23 +83,23 @@ * `hex.key remove` will now also de-auth the user if the local API key was removed * Add status messages when publishing and reverting -* Bug fixes +### Bug fixes * Fix bug where the client was fetching packages even when lock is OK * Fix resolver sometimes not producing any backtrack output * Verify certificate against correct hostname after redirect ## v0.12.1 (2016-05-31) -* Enhancements +### Enhancements * Only show proxy settings when MIX_DEBUG=1 * Add retries to idempotent requests -* Bug fixes +### Bug fixes * Fix crash when you get multiple backtrack messages ## v0.12.0 (2016-05-15) -* Enhancements +### Enhancements * Add package checksums to lock, ensuring a locked package can not change its content * Add managers and deps to lock, allowing Hex to run without loading the registry * Align deps fetching output from scm @@ -46,38 +111,38 @@ * Show app name of dependency in `hex.info` * Warn about long package descriptions -* Bug fixes +### Bug fixes * Fix `HEX_UNSAFE_HTTPS` environment variable and `unsafe_https` config ## v0.11.5 (2016-04-07) -* Enhancements +### Enhancements * Add more registry metrics to `hex.info` -* Bug fixes +### Bug fixes * Fix a bug where Hex was about a bit too enthusiastic when informing the user of new versions * Fix some missing future-proofing of lock ## v0.11.4 (2016-04-06) -* Enhancements +### Enhancements * Use HTTPS to Hex.pm repository * Make lock backwards compatible by treating it as a list and only matching on the front -* Bug fixes +### Bug fixes * Correctly show update notification * Remove duplicate parents from backtrack messages * Fix invalid message in `hex.outdated` if locked version is a pre-release ## v0.11.3 (2016-03-14) -* Bug fixes +### Bug fixes * Do not crash if registry fails to fetch * Remove force update of registry if it is more than a week old ## v0.11.2 (2016-03-11) -* Enhancements +### Enhancements * Verify registry signature against public key * Improve missing registry error message * Deprecate `HEX_CDN` in favor of `HEX_REPO` and `HEX_MIRROR`. See the `hex` task for more information @@ -86,20 +151,20 @@ * Use fastly instead of S3 for the Hex.pm repository * Add `--delete` option to `hex.config` task -* Bug fixes +### Bug fixes * Show local time in hex.info * Correctly unlock all dependencies on `deps.update` * Always fetch registry if it's missing or known to be old ## v0.11.1 (2016-03-03) -* Bug fixes +### Bug fixes * Fix incorrect build version check * Fix parsing of requirements without spaces ## v0.11.0 (2016-03-03) -* Enhancements +### Enhancements * Append the OTP version to the user_agent function * Improve output of http request timeout errors * Warn if `:manager` or `:compile` is set on dependencies when publishing @@ -112,17 +177,17 @@ * Do not allow pre-releases for dependencies unless the requirement uses a pre-release version * Optimize version cache memory usage -* Bug fixes +### Bug fixes * Fix incorrect build version check for dev versions of Elixir * Fix loop when backtracking in resolver * Fix timeout errors on slow systems ## v0.10.4 (2016-01-26) -* Enhancements +### Enhancements * Make the experimental resolver the default -* Bug fixes +### Bug fixes * Ensure registry can be opened/closed multiple times * Ensure `hex.search` task handles empty results * Fix experimental resolvers only backtracking on parents that had requirements that failed @@ -130,28 +195,28 @@ ## v0.10.3 (2016-01-23) -* Bug fixes +### Bug fixes * Fix bug when umbrella child has dependency with `:only` ## v0.10.2 (2016-01-22) -* Enhancements +### Enhancements * General optimizations in dependency resolver * Add experimental faster backtracker that does more aggressive backtracking, set environment variable `HEX_EXPERIMENTAL_RESOLVER=1` to use it * Merge backtrack messages that have similar parents * Merge multiple versions into version ranges when possible for more succinct backtrack messages -* Bug fixes +### Bug fixes * Reduce memory usage when resolver produces many backtrack messages ## v0.10.1 (2016-01-15) -* Bug fixes +### Bug fixes * Fix a crash when a dependency is missing its version requirement ## v0.10.0 (2016-01-14) -* Enhancements +### Enhancements * Add support for authentication when using HTTP proxies * Add more build information to `hex.info` task to ease debugging * Greatly improve backtracking error messages @@ -163,7 +228,7 @@ * Remove useless output when fetching dependencies * Improve package output in `hex.info` task -* Bug fixes +### Bug fixes * Fix a rare bug that could cause the resolver to go into an infinite loop * UTF8 encode package metadata * Only list missing files if `:files` is set @@ -171,7 +236,7 @@ ## v0.9.0 (2015-09-25) -* Enhancements +### Enhancements * Pass build tool information to Mix (supported in Elixir 1.1.0) * Make Hex a proper OTP application * Update CA store @@ -185,7 +250,7 @@ * Add `HEX_UNSAFE_HTTPS` for disabling certificate checking * Rename `:contributors` metadata to `:maintainers` to better reflect purpose of field -* Bug fixes +### Bug fixes * `HEX_API` no longer automatically adds `api/` to URL * Fix crash when user doesn't explicitly override Hex package when needed * Fix bug where metadata in package tarball was not properly UTF8 encoded @@ -200,85 +265,85 @@ ## v0.8.2 (2015-07-13) -* Enhancements +### Enhancements * Sort dependency resolver results -* Bug fixes +### Bug fixes * Fix build_tools metadata being sent incorrectly ## v0.8.1 (2015-07-12) -* Enhancements +### Enhancements * Warn if registry file is missing when loading deps -* Bug fixes +### Bug fixes * Consider new optional requirements for already activated dependency * Add multiple build tools to metadata ## v0.8.0 (2015-05-19) -* Enhancements +### Enhancements * Warn if using insecure SSL because of old OTP version * Use yellow test for warning text * Include build_tools in release metadata * Print more metadata when publishing -* Bug fixes +### Bug fixes * Fix an error when printing an http status codes * Always fetch new registry if it's older than 7 days ## v0.7.5 (2015-04-12) -* Enhancements +### Enhancements * Add task `hex.user test` for testing user authentication. * Add task `hex.outdated` for listing outdated packages compared to the registry. * Update CA store as of April 3. * Inform user if authentication failed because they did not confirm email. * Improve error message for unsupported tarball version. -* Bug fixes +### Bug fixes * Fix a bug where overriding a Hex dependency with a non-Hex dependency was ignored when the overriding at least two levels deep in the dependency tree ## v0.7.4 (2015-03-16) -* Bug fixes +### Bug fixes * Include all conflicting requirements in backtrack message * Fix a bug where backtrack message failed on optional requests ## v0.7.3 (2015-03-04) -* Bug fixes +### Bug fixes * Fix an error when merging locked and optional dependencies ## v0.7.2 (2015-03-04) -* Enhancements +### Enhancements * Print messages on backtracks if dependency resolution failed, this is intended to help users resolve conflicts -* Bug fixes +### Bug fixes * Fix a bug where a dependency converged in mix did not consider all its requirements * Fix a bug where dependencies in the lock was considered even if they weren't requested ## v0.7.1 (2015-02-15) -* Bug fixes +### Bug fixes * Fix updating the registry ## v0.7.0 (2015-02-15) -* Enhancements +### Enhancements * Print proxy options on startup * Add `mix hex.user password reset` and remove `mix hex.user update` * Create version 3 tarballs with erlang term encoded metadata -* Bug fixes +### Bug fixes * Verify peer certificate against CA certificate public key in `partial_chain` * Fix a bug where overriding a Hex dependency with a non-Hex dependency was ignored when the overriding happened in a sub-dependency * Create hex directory before writing registry ## v0.6.2 (2015-01-02) -* Enhancements +### Enhancements * Add PKIX hostname verification according to RFC6125 * Improve error messages from HTTP error codes * Improve HTTP performance @@ -287,24 +352,24 @@ ## v0.6.1 (2014-12-11) -* Enhancements +### Enhancements * Convert config file to erlang term file ## v0.6.0 (2014-10-12) -* Enhancements +### Enhancements * Add support for packages with a different OTP application name than the package name * Add task `mix hex.docs` for uploading project documentation * Add email confirmation -* Bug fixes +### Bug fixes * Allow you to change your password with `mix hex.user update` * Correctly display dependencies in `mix hex.info PACKAGE VERSION` * Verify peer certificates when fetching tarball ## v0.5.0 (2014-09-19) -* Enhancements +### Enhancements * Verify peer certificate for SSL (only available in OTP 17.3) * Reduce archive size with compiler option `debug_info: false` * Add support for config as an erlang term file @@ -314,23 +379,23 @@ ## v0.4.2 (2014-08-31) -* Enhancements +### Enhancements * Add task `hex.user whoami` that prints the locally authorized user * Add task `hex.user deauth` to deauthorize the local user * Rename environment variable `HEX_URL` to `HEX_API` to not confuse it with `HEX_CDN` -* Bug fixes +### Bug fixes * Print newline after progress bar ## v0.4.1 (2014-08-12) -* Enhancements +### Enhancements * Add progress bar for uploading the tarball when publishing * Compare tarball checksum against checksum in registry * Bump tarball support to version 3 * Rename task for authenticating on the local machine from `hex.key new` to `hex.user auth` * Remove the ability to pass password as a CLI parameter -* Bug fixes +### Bug fixes * Support lower-case proxy environment variables * Remove any timeouts when fetching package tarballs diff --git a/README.md b/README.md index 6d54a85a2..f6d24eacd 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Hex bundles a list of root CA certificates used for certificate validation in HT ### hex_web -Integration tests run against the API server [hex_web](https://github.com/hexpm/hex_web). It needs to be cloned into `../hex_web` or `HEX_WEB_DIR` needs to be set and point its location. hex_web also requires postgresql with username `postgres` and password `postgres`. +Integration tests run against the API server [hex_web](https://github.com/hexpm/hex_web). It needs to be cloned into `../hex_web` or `HEXWEB_PATH` needs to be set and point its location. hex_web also requires postgresql with username `postgres` and password `postgres`. Run integration tests with `mix test --include integration`. diff --git a/RELEASE.md b/RELEASE.md index 8b226a807..4e64cf067 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -38,7 +38,7 @@ Always build on the latest patch version and make sure tests pass before buildin ## Places where version is mentioned -* mix.exs `:version` option +* mix.exs `@version` attribute * CHANGELOG.md ## S3 paths diff --git a/lib/hex.ex b/lib/hex.ex index 47f194fa1..feabac8a9 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -13,7 +13,7 @@ defmodule Hex do end def start(_, _) do - import Supervisor.Spec + dev_setup() Mix.SCM.append(Hex.SCM) Mix.RemoteConverger.register(Hex.RemoteConverger) @@ -21,21 +21,15 @@ defmodule Hex do Hex.Version.start start_httpc() - children = [ - worker(Hex.State, []), - worker(Hex.Registry.ETS, []), - worker(Hex.Parallel, [:hex_fetcher, [max_parallel: 64]]), - ] - opts = [strategy: :one_for_one, name: Hex.Supervisor] - Supervisor.start_link(children, opts) + Supervisor.start_link(children(), opts) end def version, do: unquote(Mix.Project.config[:version]) def elixir_version, do: unquote(System.version) def otp_version, do: unquote(Hex.Utils.otp_version) - defp start_httpc() do + defp start_httpc do :inets.start(:httpc, profile: :hex) opts = [ max_sessions: 8, @@ -46,4 +40,37 @@ defmodule Hex do ] :httpc.set_options(opts, :hex) end + + if Version.compare(System.version, "1.3.0") == :lt do + def string_trim(string), do: String.strip(string) + def to_charlist(term), do: Kernel.to_char_list(term) + def string_to_charlist(string), do: String.to_char_list(string) + else + def string_trim(string), do: String.trim(string) + def to_charlist(term), do: Kernel.to_charlist(term) + def string_to_charlist(string), do: String.to_charlist(string) + end + + if Mix.env == :test do + defp children do + import Supervisor.Spec + [worker(Hex.State, []), + worker(Hex.Parallel, [:hex_fetcher])] + end + else + defp children do + import Supervisor.Spec + [worker(Hex.State, []), + worker(Hex.Parallel, [:hex_fetcher]), + worker(Hex.Registry.Server, [])] + end + end + + if Mix.env in [:dev, :test] do + defp dev_setup do + :erlang.system_flag(:backtrace_depth, 20) + end + else + defp dev_setup, do: :ok + end end diff --git a/lib/hex/api.ex b/lib/hex/api.ex index 30fe08b51..fd9e92f93 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -1,21 +1,11 @@ defmodule Hex.API do alias Hex.API.Utils - alias Hex.API.VerifyHostname - @request_timeout 60_000 - @secure_ssl_version {5, 3, 7} + @request_timeout 25_000 @erlang_vendor 'application/vnd.hex+erlang' - require Record - - Record.defrecordp :certificate, :OTPCertificate, - Record.extract(:OTPCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") - - Record.defrecordp :tbs_certificate, :OTPTBSCertificate, - Record.extract(:OTPTBSCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") - def request(method, url, headers, body \\ nil) - when (is_map(headers) or is_list(headers)) and (body == nil or is_map(body)) do + when (is_map(headers) or is_list(headers)) and (body == nil or is_map(body)) do default_headers = %{ 'accept' => @erlang_vendor, 'accept-encoding' => 'gzip', @@ -24,7 +14,7 @@ defmodule Hex.API do http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) opts = [body_format: :binary] - url = String.to_char_list(url) + url = Hex.string_to_charlist(url) profile = Hex.State.fetch!(:httpc_profile) request = @@ -53,7 +43,7 @@ defmodule Hex.API do url = elem(request, 0) http_opts = http_opts - |> Keyword.put(:ssl, ssl_opts(url)) + |> Keyword.put(:ssl, Hex.API.SSL.ssl_opts(url)) |> Keyword.put_new(:autoredirect, false) case :httpc.request(method, request, http_opts, opts, profile) do @@ -61,7 +51,7 @@ defmodule Hex.API do case handle_redirect(response) do {:ok, location} when times > 0 -> request = update_request(request, location) - request_with_redirect(method, request, http_opts, opts, profile, times-1) + request_with_redirect(method, request, http_opts, opts, profile, times - 1) {:ok, _location} -> Mix.raise "Too many redirects" :error -> @@ -88,69 +78,17 @@ defmodule Hex.API do end defp handle_redirect(_), do: :error - def secure_ssl? do - check? = Hex.State.fetch!(:check_cert?) - if check? and Hex.State.fetch!(:ssl_version) <= @secure_ssl_version do - Mix.raise "Insecure HTTPS request (peer verification disabled), " <> - "please update to OTP 17.4 or later, or disable by setting " <> - "the environment variable HEX_UNSAFE_HTTPS=1" - end - check? - end - - def ssl_opts(url) do - if secure_ssl?() do - url = List.to_string(url) - hostname = String.to_char_list(URI.parse(url).host) - verify_fun = {&VerifyHostname.verify_fun/3, check_hostname: hostname} - partial_chain = &partial_chain(Hex.API.Certs.cacerts, &1) - - [verify: :verify_peer, depth: 2, partial_chain: partial_chain, - cacerts: Hex.API.Certs.cacerts(), verify_fun: verify_fun, - server_name_indication: hostname] - else - [verify: :verify_none] - end - end - - def partial_chain(cacerts, certs) do - certs = Enum.map(certs, &{&1, :public_key.pkix_decode_cert(&1, :otp)}) - cacerts = Enum.map(cacerts, &:public_key.pkix_decode_cert(&1, :otp)) - - trusted = - Enum.find_value(certs, fn {der, cert} -> - trusted? = - Enum.find(cacerts, fn cacert -> - extract_public_key_info(cacert) == extract_public_key_info(cert) - end) - - if trusted?, do: der - end) - - if trusted do - {:trusted_ca, trusted} - else - :unknown_ca - end - end - - defp extract_public_key_info(cert) do - cert - |> certificate(:tbsCertificate) - |> tbs_certificate(:subjectPublicKeyInfo) - end - @chunk 10_000 def request_tar(url, headers, body, progress) do default_headers = %{ 'accept' => @erlang_vendor, 'user-agent' => user_agent(), - 'content-length' => to_char_list(byte_size(body))} + 'content-length' => Hex.to_charlist(byte_size(body))} headers = Enum.into(headers, default_headers) http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) opts = [body_format: :binary] - url = String.to_char_list(url) + url = Hex.string_to_charlist(url) profile = Hex.State.fetch!(:httpc_profile) body = fn @@ -177,7 +115,10 @@ defmodule Hex.API do headers = Enum.into(headers, %{}) Utils.handle_hex_message(headers['x-hex-message']) - body = body |> unzip(headers) |> decode(headers) + body = + body + |> unzip(headers) + |> decode(headers) {code, body, headers} end @@ -217,7 +158,7 @@ defmodule Hex.API do def auth(opts) do if key = opts[:key] do - %{'authorization' => String.to_char_list(key)} + %{'authorization' => Hex.string_to_charlist(key)} else base64 = :base64.encode_to_string(opts[:user] <> ":" <> opts[:pass]) %{'authorization' => 'Basic ' ++ base64} @@ -227,7 +168,7 @@ defmodule Hex.API do defp retry(:get, times, fun) do case fun.() do {:http_error, _, _} when times > 1 -> - retry(:get, times-1, fun) + retry(:get, times - 1, fun) {:http_error, _, _} = error -> error other -> diff --git a/lib/hex/api/ca-bundle.crt b/lib/hex/api/ca-bundle.crt index 3d84311cf..67659d63b 100644 --- a/lib/hex/api/ca-bundle.crt +++ b/lib/hex/api/ca-bundle.crt @@ -1,20 +1,20 @@ ## ## Bundle of CA Root Certificates ## -## Certificate data from Mozilla as of: Sat Jul 30 22:58:40 2016 +## Certificate data from Mozilla as of: Fri Oct 28 14:19:14 2016 GMT ## ## This is a bundle of X.509 certificates of public Certificate Authorities ## (CA). These were automatically extracted from Mozilla's root certificates ## file (certdata.txt). This file can be found in the mozilla source tree: -## http://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt +## https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt ## ## It contains the certificates in PEM format and therefore ## can be directly used with curl / libcurl / php_curl, or with ## an Apache+mod_ssl webserver for SSL client authentication. ## Just configure this file as the SSLCACertificateFile. ## -## Conversion done with mk-ca-bundle.pl version 1.25. -## SHA1: 5df367cda83086392e1acdf22bfef00c48d5eba6 +## Conversion done with mk-ca-bundle.pl version 1.27. +## SHA256: 01bbf1ecdd693f554ff4dcbe15880b3e6c33188a956c15ff845d313ca69cfeb8 ## @@ -3863,3 +3863,174 @@ ypnTycUm/Q1oBEauttmbjL4ZvrHG8hnjXALKLNhvSgfZyTXaQHXyxKcZb55CEJh15pWLYLztxRLX is7VmFxWlgPF7ncGNf/P5O4/E2Hu29othfDNrp2yGAlFw5Khchf8R7agCyzxxN5DaAhqXzvwdmP7 zAYspsbiDrW5viSP -----END CERTIFICATE----- + +Hellenic Academic and Research Institutions RootCA 2015 +======================================================= +-----BEGIN CERTIFICATE----- +MIIGCzCCA/OgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBpjELMAkGA1UEBhMCR1IxDzANBgNVBAcT +BkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0 +aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNVBAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl +YXJjaCBJbnN0aXR1dGlvbnMgUm9vdENBIDIwMTUwHhcNMTUwNzA3MTAxMTIxWhcNNDAwNjMwMTAx +MTIxWjCBpjELMAkGA1UEBhMCR1IxDzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMg +QWNhZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNV +BAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgUm9vdENBIDIw +MTUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDC+Kk/G4n8PDwEXT2QNrCROnk8Zlrv +bTkBSRq0t89/TSNTt5AA4xMqKKYx8ZEA4yjsriFBzh/a/X0SWwGDD7mwX5nh8hKDgE0GPt+sr+eh +iGsxr/CL0BgzuNtFajT0AoAkKAoCFZVedioNmToUW/bLy1O8E00BiDeUJRtCvCLYjqOWXjrZMts+ +6PAQZe104S+nfK8nNLspfZu2zwnI5dMK/IhlZXQK3HMcXM1AsRzUtoSMTFDPaI6oWa7CJ06CojXd +FPQf/7J31Ycvqm59JCfnxssm5uX+Zwdj2EUN3TpZZTlYepKZcj2chF6IIbjV9Cz82XBST3i4vTwr +i5WY9bPRaM8gFH5MXF/ni+X1NYEZN9cRCLdmvtNKzoNXADrDgfgXy5I2XdGj2HUb4Ysn6npIQf1F +GQatJ5lOwXBH3bWfgVMS5bGMSF0xQxfjjMZ6Y5ZLKTBOhE5iGV48zpeQpX8B653g+IuJ3SWYPZK2 +fu/Z8VFRfS0myGlZYeCsargqNhEEelC9MoS+L9xy1dcdFkfkR2YgP/SWxa+OAXqlD3pk9Q0Yh9mu +iNX6hME6wGkoLfINaFGq46V3xqSQDqE3izEjR8EJCOtu93ib14L8hCCZSRm2Ekax+0VVFqmjZayc +Bw/qa9wfLgZy7IaIEuQt218FL+TwA9MmM+eAws1CoRc0CwIDAQABo0IwQDAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUcRVnyMjJvXVdctA4GGqd83EkVAswDQYJKoZI +hvcNAQELBQADggIBAHW7bVRLqhBYRjTyYtcWNl0IXtVsyIe9tC5G8jH4fOpCtZMWVdyhDBKg2mF+ +D1hYc2Ryx+hFjtyp8iY/xnmMsVMIM4GwVhO+5lFc2JsKT0ucVlMC6U/2DWDqTUJV6HwbISHTGzrM +d/K4kPFox/la/vot9L/J9UUbzjgQKjeKeaO04wlshYaT/4mWJ3iBj2fjRnRUjtkNaeJK9E10A/+y +d+2VZ5fkscWrv2oj6NSU4kQoYsRL4vDY4ilrGnB+JGGTe08DMiUNRSQrlrRGar9KC/eaj8GsGsVn +82800vpzY4zvFrCopEYq+OsS7HK07/grfoxSwIuEVPkvPuNVqNxmsdnhX9izjFk0WaSrT2y7Hxjb +davYy5LNlDhhDgcGH0tGEPEVvo2FXDtKK4F5D7Rpn0lQl033DlZdwJVqwjbDG2jJ9SrcR5q+ss7F +Jej6A7na+RZukYT1HCjI/CbM1xyQVqdfbzoEvM14iQuODy+jqk+iGxI9FghAD/FGTNeqewjBCvVt +J94Cj8rDtSvK6evIIVM4pcw72Hc3MKJP2W/R8kCtQXoXxdZKNYm3QdV8hn9VTYNKpXMgwDqvkPGa +JI7ZjnHKe7iG2rKPmT4dEw0SEe7Uq/DpFXYC5ODfqiAeW2GFZECpkJcNrVPSWh2HagCXZWK0vm9q +p/UsQu0yrbYhnr68 +-----END CERTIFICATE----- + +Hellenic Academic and Research Institutions ECC RootCA 2015 +=========================================================== +-----BEGIN CERTIFICATE----- +MIICwzCCAkqgAwIBAgIBADAKBggqhkjOPQQDAjCBqjELMAkGA1UEBhMCR1IxDzANBgNVBAcTBkF0 +aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9u +cyBDZXJ0LiBBdXRob3JpdHkxRDBCBgNVBAMTO0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj +aCBJbnN0aXR1dGlvbnMgRUNDIFJvb3RDQSAyMDE1MB4XDTE1MDcwNzEwMzcxMloXDTQwMDYzMDEw +MzcxMlowgaoxCzAJBgNVBAYTAkdSMQ8wDQYDVQQHEwZBdGhlbnMxRDBCBgNVBAoTO0hlbGxlbmlj +IEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgQ2VydC4gQXV0aG9yaXR5MUQwQgYD +VQQDEztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0dXRpb25zIEVDQyBSb290 +Q0EgMjAxNTB2MBAGByqGSM49AgEGBSuBBAAiA2IABJKgQehLgoRc4vgxEZmGZE4JJS+dQS8KrjVP +dJWyUWRrjWvmP3CV8AVER6ZyOFB2lQJajq4onvktTpnvLEhvTCUp6NFxW98dwXU3tNf6e3pCnGoK +Vlp8aQuqgAkkbH7BRqNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0O +BBYEFLQiC4KZJAEOnLvkDv2/+5cgk5kqMAoGCCqGSM49BAMCA2cAMGQCMGfOFmI4oqxiRaeplSTA +GiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7SofTUwJCA3sS61kFyjn +dc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR +-----END CERTIFICATE----- + +Certplus Root CA G1 +=================== +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUAMD4xCzAJBgNV +BAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMTAe +Fw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhD +ZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHN +r49aiZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt6kuJPKNx +Qv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP0FG7Yn2ksYyy/yARujVj +BYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTv +LRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDEEW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2 +z4QTd28n6v+WZxcIbekN1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc +4nBvCGrch2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCTmehd +4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV4EJQeIQEQWGw9CEj +jy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPOWftwenMGE9nTdDckQQoRb5fc5+R+ +ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBSowcCbkahDFXxdBie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHY +lwuBsTANBgkqhkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh +66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7/SMNkPX0XtPG +YX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BSS7CTKtQ+FjPlnsZlFT5kOwQ/ +2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F +6ALEUz65noe8zDUa3qHpimOHZR4RKttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilX +CNQ314cnrUlZp5GrRHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWe +tUNy6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEVV/xuZDDC +VRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5g4VCXA9DO2pJNdWY9BW/ ++mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl++O/QmueD6i9a5jc2NvLi6Td11n0bt3+ +qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= +-----END CERTIFICATE----- + +Certplus Root CA G2 +=================== +-----BEGIN CERTIFICATE----- +MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4xCzAJBgNVBAYT +AkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMjAeFw0x +NDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0 +cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IA +BM0PW1aC3/BFGtat93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uN +Am8xIk0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMB8GA1Ud +IwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqGSM49BAMDA2gAMGUCMHD+sAvZ94OX7PNV +HdTcswYO/jOYnYs5kGuUIe22113WTNchp+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjl +vPl5adytRSv3tjFzzAalU5ORGpOucGpnutee5WEaXw== +-----END CERTIFICATE----- + +OpenTrust Root CA G1 +==================== +-----BEGIN CERTIFICATE----- +MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUAMEAxCzAJBgNV +BAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5UcnVzdCBSb290IENBIEcx +MB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAwMFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoM +CU9wZW5UcnVzdDEdMBsGA1UEAwwUT3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7fa +Yp6bwiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX/uMftk87 +ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR077F9jAHiOH3BX2pfJLKO +YheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGPuY4zbGneWK2gDqdkVBFpRGZPTBKnjix9 +xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLxp2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO +9z0M+Yo0FMT7MzUj8czxKselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq +3ywgsNw2TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+WG+Oi +n6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPwvFEVVJSmdz7QdFG9 +URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYYEQRVzXR7z2FwefR7LFxckvzluFqr +TJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUl0YhVyE12jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/Px +N3DlCPaTKbYwDQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E +PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kfgLMtMrpkZ2Cv +uVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbSFXJfLkur1J1juONI5f6ELlgK +n0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLh +X4SPgPL0DTatdrOjteFkdjpY3H1PXlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80 +nR14SohWZ25g/4/Ii+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcm +GS3tTAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L9109S5zvE/ +bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/KyPu1svf0OnWZzsD2097+o +4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJAwSQiumPv+i2tCqjI40cHLI5kqiPAlxA +OXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj1oxx +-----END CERTIFICATE----- + +OpenTrust Root CA G2 +==================== +-----BEGIN CERTIFICATE----- +MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUAMEAxCzAJBgNV +BAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5UcnVzdCBSb290IENBIEcy +MB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoM +CU9wZW5UcnVzdDEdMBsGA1UEAwwUT3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+ +Ntmh/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78eCbY2albz +4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/61UWY0jUJ9gNDlP7ZvyCV +eYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fEFY8ElggGQgT4hNYdvJGmQr5J1WqIP7wt +UdGejeBSzFfdNTVY27SPJIjki9/ca1TSgSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz +3GIZ38i1MH/1PCZ1Eb3XG7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj +3CzMpSZyYhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaHvGOz +9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4t/bQWVyJ98LVtZR0 +0dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/gh7PU3+06yzbXfZqfUAkBXKJOAGT +y3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUajn6QiL35okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59 +M4PLuG53hq8wDQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz +Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0nXGEL8pZ0keI +mUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qTRmTFAHneIWv2V6CG1wZy7HBG +S4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpTwm+bREx50B1ws9efAvSyB7DH5fitIw6mVskp +EndI2S9G/Tvw/HRwkqWOOAgfZDC2t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ +6e18CL13zSdkzJTaTkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97kr +gCf2o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU3jg9CcCo +SmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eAiN1nE28daCSLT7d0geX0 +YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14fWKGVyasvc0rQLW6aWQ9VGHgtPFGml4vm +u7JwqkwR3v98KzfUetF3NI/n+UL3PIEMS1IK +-----END CERTIFICATE----- + +OpenTrust Root CA G3 +==================== +-----BEGIN CERTIFICATE----- +MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAxCzAJBgNVBAYT +AkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5UcnVzdCBSb290IENBIEczMB4X +DTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9w +ZW5UcnVzdDEdMBsGA1UEAwwUT3BlblRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAARK7liuTcpm3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5B +ta1doYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4GA1UdDwEB +/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAf +BgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAKBggqhkjOPQQDAwNpADBmAjEAj6jcnboM +BBf6Fek9LykBl7+BFjNAk2z8+e2AcG+qj9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta +3U1fJAuwACEl74+nBCZx4nxp5V2a+EEfOzmTk51V6s2N8fvB +-----END CERTIFICATE----- diff --git a/lib/hex/api/package.ex b/lib/hex/api/package.ex index 8cff779c3..1bb2c6d10 100644 --- a/lib/hex/api/package.ex +++ b/lib/hex/api/package.ex @@ -5,6 +5,10 @@ defmodule Hex.API.Package do API.request(:get, API.api_url("packages/#{name}"), []) end + def search(search) do + API.request(:get, API.api_url("packages?search=#{search}&sort=downloads"), []) + end + defmodule Owner do def add(package, owner, auth) do owner = URI.encode_www_form(owner) diff --git a/lib/hex/api/registry.ex b/lib/hex/api/registry.ex index e91ac191f..2785de7ac 100644 --- a/lib/hex/api/registry.ex +++ b/lib/hex/api/registry.ex @@ -1,16 +1,86 @@ defmodule Hex.API.Registry do alias Hex.API + alias Hex.Crypto.PublicKey - def get(opts \\ []) do + @default_repo "https://repo.hex.pm" + @public_keys_html "https://hex.pm/docs/public_keys" + + def get_package(name, opts \\ []) do headers = if etag = opts[:etag] do - %{'if-none-match' => '"' ++ etag ++ '"'} + %{'if-none-match' => Hex.string_to_charlist(etag)} end - API.request(:get, API.repo_url("registry.ets.gz"), headers || []) + API.request(:get, API.repo_url("packages/#{name}"), headers || []) + end + + def verify(body) do + %{signature: signature, payload: payload} = :hex_pb_signed.decode_msg(body, :Signed) + if Hex.State.fetch!(:check_registry?) do + do_verify(payload, signature) + end + payload + end + + def get_installs do + API.request(:get, API.repo_url("installs/hex-1.x.csv"), []) + end + + def find_new_version_from_csv(body) do + body + |> parse_csv + |> find_latest_eligible_version + |> is_version_newer + end + + defp parse_csv(body) do + body + |> :binary.split("\n", [:global, :trim]) + |> Enum.map(&:binary.split(&1, ",", [:global, :trim])) + end + + defp find_latest_eligible_version(entries) do + elixir_version = Hex.Version.parse!(System.version) + entries + |> Enum.reverse + |> Enum.find_value(&find_version(&1, elixir_version)) + end + + defp find_version([hex_version, _digest | compatible_versions], elixir_version) do + if Enum.find(compatible_versions, &Hex.Version.compare(&1, elixir_version) != :gt) do + hex_version + end + end + + defp is_version_newer(nil), do: nil + defp is_version_newer(hex_version) do + if Hex.Version.compare(hex_version, Hex.version) == :gt do + hex_version + end + end + + defp do_verify(payload, signature) do + repo = Hex.State.fetch!(:repo) || @default_repo + key = PublicKey.public_key(repo) + + unless key do + Mix.raise "No public key stored for #{repo}. Either install a public " <> + "key with `mix hex.public_keys` or disable the registry " <> + "verification check by setting `HEX_UNSAFE_REGISTRY=1`." + end + + unless Hex.Crypto.PublicKey.verify(payload, :sha512, signature, [key]) do + Mix.raise "Could not verify authenticity of fetched registry file. " <> + "This may happen because a proxy or some entity is " <> + "interfering with the download or because you don't have a " <> + "public key to verify the registry.\n\nYou may try again " <> + "later or check if a new public key has been released on " <> + "our public keys page: #{@public_keys_html}" + end end - def get_signature() do - API.request(:get, API.repo_url("registry.ets.gz.signed"), []) + def decode(body) do + %{releases: releases} = :hex_pb_package.decode_msg(body, :Package) + releases end end diff --git a/lib/hex/api/release.ex b/lib/hex/api/release.ex index 3564f9e85..7fa1ea070 100644 --- a/lib/hex/api/release.ex +++ b/lib/hex/api/release.ex @@ -15,4 +15,14 @@ defmodule Hex.API.Release do url = API.api_url("packages/#{name}/releases/#{version}") API.request(:delete, url, API.auth(auth)) end + + def retire(name, version, body, auth) do + url = API.api_url("packages/#{name}/releases/#{version}/retire") + API.request(:post, url, API.auth(auth), body) + end + + def unretire(name, version, auth) do + url = API.api_url("packages/#{name}/releases/#{version}/retire") + API.request(:delete, url, API.auth(auth)) + end end diff --git a/lib/hex/api/ssl.ex b/lib/hex/api/ssl.ex new file mode 100644 index 000000000..2563cefb2 --- /dev/null +++ b/lib/hex/api/ssl.ex @@ -0,0 +1,106 @@ +defmodule Hex.API.SSL do + require Record + alias Hex.API.VerifyHostname + + # From https://www.ssllabs.com/ssltest/clients.html Android 7 + @default_ciphers [ + 'ECDHE-ECDSA-CHACHA20-POLY1305-SHA256', + 'ECDHE-RSA-CHACHA20-POLY1305-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-ECDSA-AES256-SHA', + 'ECDHE-RSA-AES256-SHA', + 'AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'AES128-SHA', + 'AES256-SHA', + 'DES-CBC3-SHA' + ] + + @default_versions [:"tlsv1.2", :"tlsv1.1", :tlsv1] + + @secure_ssl_version {5, 3, 7} + + Record.defrecordp :certificate, :OTPCertificate, + Record.extract(:OTPCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") + + Record.defrecordp :tbs_certificate, :OTPTBSCertificate, + Record.extract(:OTPTBSCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") + + def secure_ssl? do + check? = Hex.State.fetch!(:check_cert?) + if check? and Hex.State.fetch!(:ssl_version) <= @secure_ssl_version do + Mix.raise "Insecure HTTPS request (peer verification disabled), " <> + "please update to OTP 17.4 or later, or disable by setting " <> + "the environment variable HEX_UNSAFE_HTTPS=1" + end + check? + end + + def ssl_opts(url) do + url = List.to_string(url) + hostname = Hex.string_to_charlist(URI.parse(url).host) + ciphers = filter_ciphers(@default_ciphers) + + if secure_ssl?() do + verify_fun = {&VerifyHostname.verify_fun/3, check_hostname: hostname} + partial_chain = &partial_chain(Hex.API.Certs.cacerts, &1) + + [verify: :verify_peer, + depth: 4, + partial_chain: partial_chain, + cacerts: Hex.API.Certs.cacerts(), + verify_fun: verify_fun, + server_name_indication: hostname, + secure_renegotiate: true, + reuse_sessions: true, + honor_cipher_order: true, + versions: @default_versions, + ciphers: ciphers] + else + [verify: :verify_none, + server_name_indication: hostname, + secure_renegotiate: true, + reuse_sessions: true, + honor_cipher_order: true, + versions: @default_versions, + ciphers: ciphers] + end + end + + def partial_chain(cacerts, certs) do + certs = Enum.map(certs, &{&1, :public_key.pkix_decode_cert(&1, :otp)}) + cacerts = Enum.map(cacerts, &:public_key.pkix_decode_cert(&1, :otp)) + + trusted = + Enum.find_value(certs, fn {der, cert} -> + trusted? = + Enum.find(cacerts, fn cacert -> + extract_public_key_info(cacert) == extract_public_key_info(cert) + end) + + if trusted?, do: der + end) + + if trusted do + {:trusted_ca, trusted} + else + :unknown_ca + end + end + + defp extract_public_key_info(cert) do + cert + |> certificate(:tbsCertificate) + |> tbs_certificate(:subjectPublicKeyInfo) + end + + defp filter_ciphers(allowed) do + available = Hex.Set.new(:ssl.cipher_suites(:openssl)) + Enum.filter(allowed, &(&1 in available)) + end +end diff --git a/lib/hex/api/user.ex b/lib/hex/api/user.ex index bca3d66af..60f0ef624 100644 --- a/lib/hex/api/user.ex +++ b/lib/hex/api/user.ex @@ -1,8 +1,12 @@ defmodule Hex.API.User do alias Hex.API - def get(username, auth) do - API.request(:get, API.api_url("users/#{username}"), API.auth(auth)) + def get(username) do + API.request(:get, API.api_url("users/#{username}"), []) + end + + def test(username, auth) do + API.request(:get, API.api_url("users/#{username}/test"), API.auth(auth)) end def new(username, email, password) do diff --git a/lib/hex/api/verify_hostname.ex b/lib/hex/api/verify_hostname.ex index ac8b05a4b..c9d53d92e 100644 --- a/lib/hex/api/verify_hostname.ex +++ b/lib/hex/api/verify_hostname.ex @@ -49,25 +49,29 @@ defmodule Hex.API.VerifyHostname do def validate_and_parse_wildcard_identifier(identifier, hostname) do wildcard_pos = :string.chr(identifier, ?*) - - if wildcard_pos != 0 do - if length(hostname) >= length(identifier) do - if check_wildcard_in_leftmost_label(identifier, wildcard_pos) do - before_w = :string.substr(identifier, 1, wildcard_pos - 1) - after_w = :string.substr(identifier, wildcard_pos + 1) - - if :string.chr(after_w, ?*) == 0 do - case check_two_labels_after_wildcard(after_w) do - {:ok, dot_after_wildcard} -> - single_char_w = dot_after_wildcard == wildcard_pos and length(before_w) == 0 - {before_w, after_w, single_char_w} - :error -> - false - end - end + valid? = + wildcard_pos != 0 and + length(hostname) >= length(identifier) and + check_wildcard_in_leftmost_label(identifier, wildcard_pos) + + if valid? do + before_w = :string.substr(identifier, 1, wildcard_pos - 1) + after_w = :string.substr(identifier, wildcard_pos + 1) + + if :string.chr(after_w, ?*) == 0 do + case check_two_labels_after_wildcard(after_w) do + {:ok, dot_after_wildcard} -> + single_char_w = dot_after_wildcard == wildcard_pos and length(before_w) == 0 + {before_w, after_w, single_char_w} + :error -> + false end + else + false end - end || false + else + false + end end def try_match_hostname(identifier, hostname) do @@ -173,6 +177,10 @@ defmodule Hex.API.VerifyHostname do defp maybe_check_subject_cn([_|_], false, _tbs_cert, _hostname), do: false - defp maybe_check_subject_cn(_dns_names, false, tbs_cert, hostname), - do: otp_tbs_certificate(tbs_cert, :subject) |> extract_cn |> try_match_hostname(hostname) + defp maybe_check_subject_cn(_dns_names, false, tbs_cert, hostname) do + tbs_cert + |> otp_tbs_certificate(:subject) + |> extract_cn + |> try_match_hostname(hostname) + end end diff --git a/lib/hex/crypto/aes_cbc_hmac_sha2.ex b/lib/hex/crypto/aes_cbc_hmac_sha2.ex index c9a9240a3..edcbd9253 100644 --- a/lib/hex/crypto/aes_cbc_hmac_sha2.ex +++ b/lib/hex/crypto/aes_cbc_hmac_sha2.ex @@ -1,53 +1,46 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do + alias Hex.Crypto.ContentEncryptor + + @behaviour ContentEncryptor + @moduledoc ~S""" Content Encryption with AES_CBC_HMAC_SHA2. See: https://tools.ietf.org/html/rfc7518#section-5.2.6 """ - alias Hex.Crypto.ContentEncryptor - @spec content_encrypt({binary, binary}, <<_::32>> | <<_::48>> | <<_::64>>, <<_::16>>) :: {binary, <<_::16>> | <<_::24>> | <<_::32>>} def content_encrypt({aad, plain_text}, key, iv) - when is_binary(aad) - and is_binary(plain_text) - and bit_size(key) in [256, 384, 512] - and bit_size(iv) === 128 do + when is_binary(aad) and + is_binary(plain_text) and + bit_size(key) in [256, 384, 512] and + bit_size(iv) === 128 do mac_size = div(byte_size(key), 2) enc_size = mac_size tag_size = mac_size - << - mac_key :: binary-size(mac_size), - enc_key :: binary-size(enc_size) - >> = key + <> = key cipher_text = aes_cbc_encrypt(enc_key, iv, pkcs7_pad(plain_text)) - aad_length = << (bit_size(aad)) :: 1-unsigned-big-integer-unit(64) >> + aad_length = <> mac_data = aad <> iv <> cipher_text <> aad_length - << - cipher_tag :: binary-size(tag_size), - _ :: binary - >> = hmac_sha2(mac_key, mac_data) + <> = hmac_sha2(mac_key, mac_data) {cipher_text, cipher_tag} end @spec content_decrypt({binary, binary, <<_::16>> | <<_::24>> | <<_::32>>}, <<_::32>> | <<_::48>> | <<_::64>>, <<_::16>>) :: {:ok, binary} | :error def content_decrypt({aad, cipher_text, cipher_tag}, key, iv) - when is_binary(aad) - and is_binary(cipher_text) - and bit_size(cipher_tag) in [128, 192, 256] - and bit_size(key) in [256, 384, 512] - and bit_size(iv) === 128 do + when is_binary(aad) and + is_binary(cipher_text) and + bit_size(cipher_tag) in [128, 192, 256] and + bit_size(key) in [256, 384, 512] and + bit_size(iv) === 128 do mac_size = div(byte_size(key), 2) enc_size = mac_size tag_size = mac_size - << - mac_key :: binary-size(mac_size), - enc_key :: binary-size(enc_size) - >> = key - aad_length = << (bit_size(aad)) :: 1-unsigned-big-integer-unit(64) >> + <> = key + aad_length = <> mac_data = aad <> iv <> cipher_text <> aad_length case hmac_sha2(mac_key, mac_data) do - << ^cipher_tag :: binary-size(tag_size), _ :: binary >> -> + <<^cipher_tag::binary-size(tag_size), _::binary>> -> case aes_cbc_decrypt(enc_key, iv, cipher_text) do plain_text when is_binary(plain_text) -> pkcs7_unpad(plain_text) @@ -59,29 +52,17 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do end end - ## Content Encryptor - - @behaviour ContentEncryptor - - def init(%{ enc: enc }, _options) - when enc in ["A128CBC-HS256", "A192CBC-HS384", "A256CBC-HS512"] do - key_length = - case enc do - "A128CBC-HS256" -> 32 - "A192CBC-HS384" -> 48 - "A256CBC-HS512" -> 64 - end - params = %{ - key_length: key_length - } - {:ok, params} + def init(%{enc: enc}, _opts) do + {:ok, %{key_length: encoding_to_key_length(enc)}} end - def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) when byte_size(key) == key_length do + def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) + when byte_size(key) == key_length do content_encrypt({aad, plain_text}, key, iv) end - def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) when byte_size(key) == key_length do + def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) + when byte_size(key) == key_length do content_decrypt({aad, cipher_text, cipher_tag}, key, iv) end @@ -97,38 +78,26 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do key_length end - ## Internal - # Support new and old style AES-CBC calls. defp aes_cbc_encrypt(key, iv, plain_text) do - try do - :crypto.block_encrypt(:aes_cbc, key, iv, plain_text) - catch - _,_ -> - cipher = - case bit_size(key) do - 128 -> :aes_cbc128 - 192 -> :aes_cbc192 - 256 -> :aes_cbc256 - end - :crypto.block_encrypt(cipher, key, iv, plain_text) - end + :crypto.block_encrypt(:aes_cbc, key, iv, plain_text) + rescue + FunctionClauseError -> + key + |> bit_size() + |> bit_size_to_cipher() + |> :crypto.block_encrypt(key, iv, plain_text) end # Support new and old style AES-CBC calls. defp aes_cbc_decrypt(key, iv, cipher_text) do - try do - :crypto.block_decrypt(:aes_cbc, key, iv, cipher_text) - catch - _,_ -> - cipher = - case bit_size(key) do - 128 -> :aes_cbc128 - 192 -> :aes_cbc192 - 256 -> :aes_cbc256 - end - :crypto.block_decrypt(cipher, key, iv, cipher_text) - end + :crypto.block_decrypt(:aes_cbc, key, iv, cipher_text) + rescue + FunctionClauseError -> + key + |> bit_size() + |> bit_size_to_cipher() + |> :crypto.block_decrypt(key, iv, cipher_text) end defp hmac_sha2(mac_key, mac_data) when bit_size(mac_key) === 128, @@ -147,9 +116,9 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do padding_size = 16 - bytes_remaining message <> :binary.copy(<>, padding_size) end - + # Unpads a message using the PKCS #7 cryptographic message syntax. - # + # # See: https://tools.ietf.org/html/rfc2315 # See: `pkcs7_pad/1` defp pkcs7_unpad(<<>>), @@ -168,4 +137,11 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do end end -end \ No newline at end of file + defp encoding_to_key_length("A128CBC-HS256"), do: 32 + defp encoding_to_key_length("A192CBC-HS384"), do: 48 + defp encoding_to_key_length("A256CBC-HS512"), do: 64 + + defp bit_size_to_cipher(128), do: :aes_cbc128 + defp bit_size_to_cipher(192), do: :aes_cbc192 + defp bit_size_to_cipher(256), do: :aes_cbc256 +end diff --git a/lib/hex/crypto/aes_gcm.ex b/lib/hex/crypto/aes_gcm.ex index b67dfce53..5481e3180 100644 --- a/lib/hex/crypto/aes_gcm.ex +++ b/lib/hex/crypto/aes_gcm.ex @@ -1,4 +1,8 @@ defmodule Hex.Crypto.AES_GCM do + alias Hex.Crypto.ContentEncryptor + + @behaviour ContentEncryptor + @moduledoc ~S""" Content Encryption with AES GCM @@ -6,24 +10,22 @@ defmodule Hex.Crypto.AES_GCM do See: http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf """ - alias Hex.Crypto.ContentEncryptor - @spec content_encrypt({binary, binary}, <<_::16>> | <<_::24>> | <<_::32>>, <<_::12>>) :: {binary, <<_::16>>} def content_encrypt({aad, plain_text}, key, iv) - when is_binary(aad) - and is_binary(plain_text) - and bit_size(key) in [128, 192, 256] - and bit_size(iv) === 96 do + when is_binary(aad) and + is_binary(plain_text) and + bit_size(key) in [128, 192, 256] and + bit_size(iv) === 96 do :crypto.block_encrypt(:aes_gcm, key, iv, {aad, plain_text}) end @spec content_decrypt({binary, binary, <<_::16>>}, <<_::16>> | <<_::24>> | <<_::32>>, <<_::12>>) :: {:ok, binary} | :error def content_decrypt({aad, cipher_text, cipher_tag}, key, iv) - when is_binary(aad) - and is_binary(cipher_text) - and bit_size(cipher_tag) === 128 - and bit_size(key) in [128, 192, 256] - and bit_size(iv) === 96 do + when is_binary(aad) and + is_binary(cipher_text) and + bit_size(cipher_tag) === 128 and + bit_size(key) in [128, 192, 256] and + bit_size(iv) === 96 do case :crypto.block_decrypt(:aes_gcm, key, iv, {aad, cipher_text, cipher_tag}) do plain_text when is_binary(plain_text) -> {:ok, plain_text} @@ -32,29 +34,17 @@ defmodule Hex.Crypto.AES_GCM do end end - ## Content Encryptor - - @behaviour ContentEncryptor - - def init(%{ enc: enc }, _options) - when enc in ["A128GCM", "A192GCM", "A256GCM"] do - key_length = - case enc do - "A128GCM" -> 16 - "A192GCM" -> 24 - "A256GCM" -> 32 - end - params = %{ - key_length: key_length - } - {:ok, params} + def init(%{enc: enc}, _opts) do + {:ok, %{key_length: encoding_to_key_length(enc)}} end - def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) when byte_size(key) == key_length do + def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) + when byte_size(key) == key_length do content_encrypt({aad, plain_text}, key, iv) end - def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) when byte_size(key) == key_length do + def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) + when byte_size(key) == key_length do content_decrypt({aad, cipher_text, cipher_tag}, key, iv) end @@ -70,4 +60,7 @@ defmodule Hex.Crypto.AES_GCM do key_length end -end \ No newline at end of file + defp encoding_to_key_length("A128GCM"), do: 16 + defp encoding_to_key_length("A192GCM"), do: 24 + defp encoding_to_key_length("A256GCM"), do: 32 +end diff --git a/lib/hex/crypto/content_encryptor.ex b/lib/hex/crypto/content_encryptor.ex index 76f1825db..c6af5e5a5 100644 --- a/lib/hex/crypto/content_encryptor.ex +++ b/lib/hex/crypto/content_encryptor.ex @@ -1,8 +1,8 @@ defmodule Hex.Crypto.ContentEncryptor do - alias Hex.Crypto + alias __MODULE__ - @type t :: %__MODULE__{ + @type t :: %ContentEncryptor{ module: module, params: any } @@ -12,32 +12,32 @@ defmodule Hex.Crypto.ContentEncryptor do params: nil ] - @callback init(protected :: map, options :: Keyword.t) - :: {:ok, any} | {:error, String.t} + @callback init(protected :: map, options :: Keyword.t) :: + {:ok, any} | {:error, String.t} - @callback encrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, plain_text :: binary}) - :: {binary, binary} + @callback encrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, plain_text :: binary}) :: + {binary, binary} - @callback decrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, cipher_text :: binary, cipher_tag :: binary}) - :: {:ok, binary} | :error + @callback decrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, cipher_text :: binary, cipher_tag :: binary}) :: + {:ok, binary} | :error - @callback generate_key(params :: any) - :: binary + @callback generate_key(params :: any) :: + binary - @callback generate_iv(params :: any) - :: binary + @callback generate_iv(params :: any) :: + binary - @callback key_length(params :: any) - :: non_neg_integer + @callback key_length(params :: any) :: + non_neg_integer - def init(protected = %{ enc: enc }, options) do + def init(protected = %{enc: enc}, opts) do case content_encryptor_module(enc) do :error -> {:error, "Unrecognized ContentEncryptor algorithm: #{inspect enc}"} module -> - case module.init(protected, options) do + case module.init(protected, opts) do {:ok, params} -> - content_encryptor = %__MODULE__{module: module, params: params} + content_encryptor = %ContentEncryptor{module: module, params: params} {:ok, content_encryptor} content_encryptor_error -> content_encryptor_error @@ -45,28 +45,26 @@ defmodule Hex.Crypto.ContentEncryptor do end end - def encrypt(%__MODULE__{module: module, params: params}, key, iv, {aad, plain_text}) do + def encrypt(%ContentEncryptor{module: module, params: params}, key, iv, {aad, plain_text}) do module.encrypt(params, key, iv, {aad, plain_text}) end - def decrypt(%__MODULE__{module: module, params: params}, key, iv, {aad, cipher_text, cipher_tag}) do + def decrypt(%ContentEncryptor{module: module, params: params}, key, iv, {aad, cipher_text, cipher_tag}) do module.decrypt(params, key, iv, {aad, cipher_text, cipher_tag}) end - def generate_key(%__MODULE__{module: module, params: params}) do + def generate_key(%ContentEncryptor{module: module, params: params}) do module.generate_key(params) end - def generate_iv(%__MODULE__{module: module, params: params}) do + def generate_iv(%ContentEncryptor{module: module, params: params}) do module.generate_iv(params) end - def key_length(%__MODULE__{module: module, params: params}) do + def key_length(%ContentEncryptor{module: module, params: params}) do module.key_length(params) end - ## Internal - defp content_encryptor_module("A128CBC-HS256"), do: Crypto.AES_CBC_HMAC_SHA2 defp content_encryptor_module("A192CBC-HS384"), do: Crypto.AES_CBC_HMAC_SHA2 defp content_encryptor_module("A256CBC-HS512"), do: Crypto.AES_CBC_HMAC_SHA2 @@ -74,5 +72,4 @@ defmodule Hex.Crypto.ContentEncryptor do defp content_encryptor_module("A192GCM"), do: Crypto.AES_GCM defp content_encryptor_module("A256GCM"), do: Crypto.AES_GCM defp content_encryptor_module(_), do: :error - -end \ No newline at end of file +end diff --git a/lib/hex/crypto/encryption.ex b/lib/hex/crypto/encryption.ex index 5a177f6a1..d45b3a7f7 100644 --- a/lib/hex/crypto/encryption.ex +++ b/lib/hex/crypto/encryption.ex @@ -1,5 +1,4 @@ defmodule Hex.Crypto.Encryption do - alias Hex.Crypto alias Hex.Crypto.ContentEncryptor alias Hex.Crypto.KeyManager @@ -10,44 +9,37 @@ defmodule Hex.Crypto.Encryption do iv = ContentEncryptor.generate_iv(content_encryptor) protected = :erlang.term_to_binary(protected) aad = tag <> protected - {cipher_text, cipher_tag} = - ContentEncryptor.encrypt(content_encryptor, key, iv, {aad, plain_text}) - %{ - protected: protected, + {cipher_text, cipher_tag} = ContentEncryptor.encrypt(content_encryptor, key, iv, {aad, plain_text}) + %{protected: protected, encrypted_key: encrypted_key, - iv: iv, - cipher_text: cipher_text, - cipher_tag: cipher_tag - } - |> :erlang.term_to_binary() - |> Crypto.base64url_encode() + iv: iv, + cipher_text: cipher_text, + cipher_tag: cipher_tag} + |> :erlang.term_to_binary + |> Crypto.base64url_encode encrypt_init_error -> encrypt_init_error end end def decrypt({tag, cipher_text}, options) do - try do - {:ok, cipher_text} = Crypto.base64url_decode(cipher_text) - %{ - protected: protected, - encrypted_key: encrypted_key, - iv: iv, - cipher_text: cipher_text, - cipher_tag: cipher_tag - } = :erlang.binary_to_term(cipher_text, [:safe]) - aad = tag <> protected - protected = :erlang.binary_to_term(protected, [:safe]) - case KeyManager.decrypt(protected, encrypted_key, options) do - {:ok, key, content_encryptor} -> - ContentEncryptor.decrypt(content_encryptor, key, iv, {aad, cipher_text, cipher_tag}) - decrypt_init_error -> - decrypt_init_error - end - catch - _,_ -> - :error + {:ok, cipher_text} = Crypto.base64url_decode(cipher_text) + %{protected: protected, + encrypted_key: encrypted_key, + iv: iv, + cipher_text: cipher_text, + cipher_tag: cipher_tag} = :erlang.binary_to_term(cipher_text, [:safe]) + aad = tag <> protected + protected = :erlang.binary_to_term(protected, [:safe]) + case KeyManager.decrypt(protected, encrypted_key, options) do + {:ok, key, content_encryptor} -> + ContentEncryptor.decrypt(content_encryptor, key, iv, {aad, cipher_text, cipher_tag}) + decrypt_init_error -> + decrypt_init_error end + rescue + ArgumentError -> + :error end -end \ No newline at end of file +end diff --git a/lib/hex/crypto/key_manager.ex b/lib/hex/crypto/key_manager.ex index db17a9577..78ad4206d 100644 --- a/lib/hex/crypto/key_manager.ex +++ b/lib/hex/crypto/key_manager.ex @@ -1,9 +1,9 @@ defmodule Hex.Crypto.KeyManager do - alias Hex.Crypto alias Hex.Crypto.ContentEncryptor + alias __MODULE__ - @type t :: %__MODULE__{ + @type t :: %KeyManager{ module: module, params: any } @@ -13,38 +13,33 @@ defmodule Hex.Crypto.KeyManager do params: nil ] - @callback init(protected :: map, options :: Keyword.t) - :: {:ok, any} | {:error, String.t} + @callback init(protected :: map, options :: Keyword.t) :: + {:ok, any} | {:error, String.t} - @callback encrypt(params :: any, protected :: map, content_encryptor :: ContentEncryptor.t) - :: {:ok, map, binary, binary} | {:error, String.t} + @callback encrypt(params :: any, protected :: map, content_encryptor :: ContentEncryptor.t) :: + {:ok, map, binary, binary} | {:error, String.t} - @callback decrypt(params :: any, protected :: map, encrypted_key :: binary, content_encryptor :: ContentEncryptor.t) - :: {:ok, binary} | {:error, String.t} + @callback decrypt(params :: any, protected :: map, encrypted_key :: binary, content_encryptor :: ContentEncryptor.t) :: + {:ok, binary} | {:error, String.t} - def init(protected = %{ alg: alg }, options) do + def init(%{alg: alg} = protected, opts) do case key_manager_module(alg) do - :error -> - {:error, "Unrecognized KeyManager algorithm: #{inspect alg}"} - module -> - case module.init(protected, options) do + {:ok, module} -> + case module.init(protected, opts) do {:ok, params} -> - key_manager = %__MODULE__{module: module, params: params} - case ContentEncryptor.init(protected, options) do - {:ok, content_encryptor} -> - {:ok, key_manager, content_encryptor} - content_encryptor_error -> - content_encryptor_error - end + key_manager = %KeyManager{module: module, params: params} + fetch_content_encryptor(key_manager, protected, opts) key_manager_error -> key_manager_error end + error -> + error end end - def encrypt(protected, options) do - case init(protected, options) do - {:ok, %__MODULE__{module: module, params: params}, content_encryptor} -> + def encrypt(protected, opts) do + case init(protected, opts) do + {:ok, %KeyManager{module: module, params: params}, content_encryptor} -> case module.encrypt(params, protected, content_encryptor) do {:ok, protected, key, encrypted_key} -> {:ok, protected, key, encrypted_key, content_encryptor} @@ -56,9 +51,9 @@ defmodule Hex.Crypto.KeyManager do end end - def decrypt(protected, encrypted_key, options) do - case init(protected, options) do - {:ok, %__MODULE__{module: module, params: params}, content_encryptor} -> + def decrypt(protected, encrypted_key, opts) do + case init(protected, opts) do + {:ok, %KeyManager{module: module, params: params}, content_encryptor} -> case module.decrypt(params, protected, encrypted_key, content_encryptor) do {:ok, key} -> {:ok, key, content_encryptor} @@ -70,11 +65,17 @@ defmodule Hex.Crypto.KeyManager do end end - ## Internal - - defp key_manager_module("PBES2-HS256"), do: Crypto.PBES2_HMAC_SHA2 - defp key_manager_module("PBES2-HS384"), do: Crypto.PBES2_HMAC_SHA2 - defp key_manager_module("PBES2-HS512"), do: Crypto.PBES2_HMAC_SHA2 - defp key_manager_module(_), do: :error + defp key_manager_module("PBES2-HS256"), do: {:ok, Crypto.PBES2_HMAC_SHA2} + defp key_manager_module("PBES2-HS384"), do: {:ok, Crypto.PBES2_HMAC_SHA2} + defp key_manager_module("PBES2-HS512"), do: {:ok, Crypto.PBES2_HMAC_SHA2} + defp key_manager_module(alg), do: {:error, "Unrecognized KeyManager algorithm: #{inspect alg}"} -end \ No newline at end of file + defp fetch_content_encryptor(key_manager, protected, opts) do + case ContentEncryptor.init(protected, opts) do + {:ok, content_encryptor} -> + {:ok, key_manager, content_encryptor} + error -> + error + end + end +end diff --git a/lib/hex/crypto/pbes2_hmac_sha2.ex b/lib/hex/crypto/pbes2_hmac_sha2.ex index ef499a040..c3308729d 100644 --- a/lib/hex/crypto/pbes2_hmac_sha2.ex +++ b/lib/hex/crypto/pbes2_hmac_sha2.ex @@ -1,4 +1,10 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do + alias Hex.Crypto.ContentEncryptor + alias Hex.Crypto.KeyManager + alias Hex.Crypto.PKCS5 + + @behaviour KeyManager + @moduledoc ~S""" Direct Key Derivation with PBES2 and HMAC-SHA-2. @@ -6,78 +12,81 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do See: https://tools.ietf.org/html/rfc2898#section-6.2 """ - alias Hex.Crypto.ContentEncryptor - alias Hex.Crypto.KeyManager - alias Hex.Crypto.PKCS5 - @spec derive_key(String.t, binary, pos_integer, non_neg_integer, :sha256 | :sha384 | :sha512) :: binary def derive_key(password, salt_input, iterations, derived_key_length, hash) - when is_binary(password) - and is_binary(salt_input) - and is_integer(iterations) and iterations >= 1 - and is_integer(derived_key_length) and derived_key_length >= 0 - and hash in [:sha256, :sha384, :sha512] do + when is_binary(password) and + is_binary(salt_input) and + is_integer(iterations) and iterations >= 1 and + is_integer(derived_key_length) and derived_key_length >= 0 and + hash in [:sha256, :sha384, :sha512] do salt = wrap_salt_input(salt_input, hash) derived_key = PKCS5.pbkdf2(password, salt, iterations, derived_key_length, hash) derived_key end - ## Key Manager - - @behaviour KeyManager - - def init(protected = %{ alg: alg }, options) - when alg in ["PBES2-HS256", "PBES2-HS384", "PBES2-HS512"] do - hash = - case alg do - "PBES2-HS256" -> :sha256 - "PBES2-HS384" -> :sha384 - "PBES2-HS512" -> :sha512 - end - case Keyword.fetch(options, :password) do - {:ok, password} when is_binary(password) -> - case Map.fetch(protected, :p2c) do - {:ok, iterations} when is_integer(iterations) and iterations >= 1 -> - case Map.fetch(protected, :p2s) do - {:ok, salt} when is_binary(salt) -> - params = %{ - hash: hash, - password: password - } - {:ok, params} - _ -> - {:error, "protected :p2s (PBKDF2 salt) must be a binary"} - end - _ -> - {:error, "protected :p2c (PBKDF2 iterations) must be a positive integer"} + def init(%{alg: alg} = protected, opts) do + hash = algorithm_to_hash(alg) + case fetch_password(opts) do + {:ok, password} -> + case fetch_p2c(protected) do + {:ok, _iteration} -> + protected + |> fetch_p2s() + |> handle_p2s(hash, password) + error -> + error end - _ -> - {:error, "option :password (PBKDF2 password) must be a binary"} + error -> + error end end def encrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt} = protected, content_encryptor) do derived_key_length = ContentEncryptor.key_length(content_encryptor) key = derive_key(password, salt, iterations, derived_key_length, hash) - encrypted_key = <<>> + encrypted_key = "" {:ok, protected, key, encrypted_key} end - def decrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt}, <<>>, content_encryptor) do + def decrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt}, "", content_encryptor) do derived_key_length = ContentEncryptor.key_length(content_encryptor) key = derive_key(password, salt, iterations, derived_key_length, hash) {:ok, key} end - def decrypt(_, _, _, _), - do: :error + def decrypt(_, _, _, _), do: :error + + defp handle_p2s({:ok, _salt}, hash, passwd), do: {:ok, %{hash: hash, password: passwd}} + defp handle_p2s(error, _, _), do: error + + defp fetch_password(opts) do + case Keyword.fetch(opts, :password) do + {:ok, password} when is_binary(password) -> {:ok, password} + _ -> {:error, "option :password (PBKDF2 password) must be a binary"} + end + end - ## Internal + defp fetch_p2c(opts) do + case Map.fetch(opts, :p2c) do + {:ok, p2c} when is_integer(p2c) and p2c >= 1 -> {:ok, p2c} + _ -> {:error, "protected :p2c (PBKDF2 iterations) must be a positive integer"} + end + end + + defp fetch_p2s(opts) do + case Map.fetch(opts, :p2s) do + {:ok, p2s} when is_binary(p2s) -> {:ok, p2s} + _ -> {:error, "protected :p2s (PBKDF2 salt) must be a binary"} + end + end defp wrap_salt_input(salt_input, :sha256), - do: << "PBES2-HS256", 0, salt_input :: binary >> + do: <<"PBES2-HS256", 0, salt_input::binary>> defp wrap_salt_input(salt_input, :sha384), - do: << "PBES2-HS384", 0, salt_input :: binary >> + do: <<"PBES2-HS384", 0, salt_input::binary>> defp wrap_salt_input(salt_input, :sha512), - do: << "PBES2-HS512", 0, salt_input :: binary >> + do: <<"PBES2-HS512", 0, salt_input::binary>> -end \ No newline at end of file + defp algorithm_to_hash("PBES2-HS256"), do: :sha256 + defp algorithm_to_hash("PBES2-HS384"), do: :sha384 + defp algorithm_to_hash("PBES2-HS512"), do: :sha512 +end diff --git a/lib/hex/crypto/pkcs5.ex b/lib/hex/crypto/pkcs5.ex index 0ca8c95d1..d6a05e0c5 100644 --- a/lib/hex/crypto/pkcs5.ex +++ b/lib/hex/crypto/pkcs5.ex @@ -6,25 +6,21 @@ defmodule Hex.Crypto.PKCS5 do """ def pbkdf2(password, salt, iterations, derived_key_length, hash) - when is_binary(password) - and is_binary(salt) - and is_integer(iterations) and iterations >= 1 - and is_integer(derived_key_length) and derived_key_length >= 0 do + when is_binary(password) and + is_binary(salt) and + is_integer(iterations) and iterations >= 1 and + is_integer(derived_key_length) and derived_key_length >= 0 do hash_length = byte_size(:crypto.hmac(hash, <<>>, <<>>)) if derived_key_length > (0xFFFFFFFF * hash_length) do raise ArgumentError, "derived key too long" else rounds = ceildiv(derived_key_length, hash_length) - << - derived_key :: binary-size(derived_key_length), - _ :: binary - >> = pbkdf2_iterate(password, salt, iterations, hash, 1, rounds, <<>>) + <> = + pbkdf2_iterate(password, salt, iterations, hash, 1, rounds, "") derived_key end end - ## Internal - defp ceildiv(a, b) do div(a, b) + (if rem(a, b) === 0, do: 0, else: 1) end diff --git a/lib/hex/public_key.ex b/lib/hex/crypto/public_key.ex similarity index 80% rename from lib/hex/public_key.ex rename to lib/hex/crypto/public_key.ex index eda6092fd..882027770 100644 --- a/lib/hex/public_key.ex +++ b/lib/hex/crypto/public_key.ex @@ -1,4 +1,4 @@ -defmodule Hex.PublicKey do +defmodule Hex.Crypto.PublicKey do @doc """ Returns the filesystem path for public keys. """ @@ -17,11 +17,11 @@ defmodule Hex.PublicKey do [] end - keys ++ [{"hex.pm", Hex.State.fetch!(:hexpm_pk)}] + keys ++ [{"https://repo.hex.pm", Hex.State.fetch!(:hexpm_pk)}] end - def public_keys(domain) do - Enum.find(public_keys(), fn {domain2, _key} -> domain == domain2 end) + def public_key(url) do + Enum.find(public_keys(), fn {url2, _key} -> url == url2 end) end @doc """ @@ -46,7 +46,7 @@ defmodule Hex.PublicKey do """ def verify(binary, hash, signature, keys) do Enum.any?(keys, fn {id, key} -> - :public_key.verify binary, hash, signature, decode!(id, key) + :public_key.verify(binary, hash, signature, decode!(id, key)) end) end end diff --git a/lib/hex/mix.ex b/lib/hex/mix.ex index f2b6119dc..9ff8f11e1 100644 --- a/lib/hex/mix.ex +++ b/lib/hex/mix.ex @@ -92,7 +92,6 @@ defmodule Hex.Mix do end) new_dep = {app, override, children} - put_dep(deps, new_dep) ++ children end @@ -175,16 +174,54 @@ defmodule Hex.Mix do Takes a map of `{name, version}` and returns them as a lock of Hex packages. """ - @spec to_lock(%{}) :: %{} def to_lock(result) do Enum.into(result, %{}, fn {name, app, version} -> - checksum = Hex.Registry.get_checksum(name, version) |> String.downcase - managers = Hex.Registry.get_build_tools(name, version) |> Enum.map(&String.to_atom/1) - deps = Hex.Registry.get_deps(name, version) |> Enum.map(®istry_dep_to_def/1) - {String.to_atom(app), {:hex, String.to_atom(name), version, checksum, managers, deps}} + app = String.to_atom(app) + checksum = Hex.Registry.checksum(name, version) |> Base.encode16(case: :lower) + deps = + name + |> Hex.Registry.deps(version) + |> Enum.map(®istry_dep_to_def/1) + |> Enum.sort + managers = + app + |> managers() + |> Enum.sort + |> Enum.uniq + {app, {:hex, String.to_atom(name), version, checksum, managers, deps}} end) end + # We need to get managers from manifest if a dependency is not in the lock + # but it's already fetched. Without the manifest we would only get managers + # from metadata during checkout or from the lock entry. + defp managers(nil), do: [] + defp managers(app) do + path = Path.join([Mix.Project.deps_path, Atom.to_string(app), ".hex"]) + case File.read(path) do + {:ok, file} -> + case Hex.SCM.parse_manifest(file) do + {_name, _version, _checksum, managers} -> + managers + _ -> + [] + end + _ -> + [] + end + end + defp registry_dep_to_def({name, app, req, optional}), do: {String.to_atom(app), req, hex: String.to_atom(name), optional: optional} + + def packages_from_lock(lock) do + Enum.flat_map(lock, fn {_app, info} -> + case Hex.Utils.lock(info) do + [:hex, name, _version, _checksum, _managers, _deps] -> + [Atom.to_string(name)] + _ -> + [] + end + end) + end end diff --git a/lib/hex/parallel.ex b/lib/hex/parallel.ex index cc8ce52a0..68a18013d 100644 --- a/lib/hex/parallel.ex +++ b/lib/hex/parallel.ex @@ -6,13 +6,13 @@ defmodule Hex.Parallel do use GenServer - def start_link(name, opts) do - GenServer.start_link(__MODULE__, new_state(opts), name: name) + def start_link(name) do + GenServer.start_link(__MODULE__, [], name: name) end - @spec run(GenServer.server, any, (() -> any)) :: :ok - def run(name, id, fun) do - GenServer.cast(name, {:run, id, fun}) + @spec run(GenServer.server, any, Keyword.t, (() -> any)) :: :ok + def run(name, id, opts \\ [], fun) do + GenServer.call(name, {:run, id, opts, fun}) end @spec await(GenServer.server, any, timeout) :: any @@ -24,9 +24,22 @@ defmodule Hex.Parallel do GenServer.call(name, :clear) end - def handle_cast({:run, id, fun}, state) do + def init([]) do + {:ok, new_state()} + end + + def handle_call({:run, id, opts, fun}, {pid, _ref}, state) do + await? = Keyword.get(opts, :await, true) state = run_task(id, fun, state) - {:noreply, state} + + state = + if await? do + state + else + %{state | waiting_reply: Map.put(state.waiting_reply, id, {:send, pid})} + end + + {:reply, :ok, state} end def handle_call({:await, id}, from, state) do @@ -34,7 +47,7 @@ defmodule Hex.Parallel do state = %{state | finished: Map.delete(state.finished, id)} {:reply, result, state} else - state = %{state | waiting_reply: Map.put(state.waiting_reply, id, from)} + state = %{state | waiting_reply: Map.put(state.waiting_reply, id, {:gen, from})} {:noreply, state} end end @@ -53,24 +66,10 @@ defmodule Hex.Parallel do if task = Enum.find(tasks, &(&1.ref == ref)) do id = state.running[task] - state = %{state | running: Map.delete(state.running, task)} - - state = - if from = state.waiting_reply[id] do - GenServer.reply(from, message) - %{state | waiting_reply: Map.delete(state.waiting_reply, id)} - else - %{state | finished: Map.put(state.finished, id, message)} - end - state = - case :queue.out(state.waiting) do - {{:value, {id, fun}}, waiting} -> - state = %{state | waiting: waiting} - run_task(id, fun, state) - {:empty, _} -> - state - end + %{state | running: Map.delete(state.running, task)} + |> reply(id, message) + |> next_task {:noreply, state} else @@ -88,6 +87,29 @@ defmodule Hex.Parallel do end end + defp reply(state, id, message) do + case state.waiting_reply[id] do + {:gen, from} -> + GenServer.reply(from, message) + %{state | waiting_reply: Map.delete(state.waiting_reply, id)} + {:send, pid} -> + send(pid, message) + %{state | waiting_reply: Map.delete(state.waiting_reply, id)} + nil -> + %{state | finished: Map.put(state.finished, id, message)} + end + end + + defp next_task(state) do + case :queue.out(state.waiting) do + {{:value, {id, fun}}, waiting} -> + state = %{state | waiting: waiting} + run_task(id, fun, state) + {:empty, _} -> + state + end + end + defp run_task(id, fun, state) do if Map.size(state.running) >= state.max_jobs do %{state | waiting: :queue.in({id, fun}, state.waiting)} @@ -97,8 +119,8 @@ defmodule Hex.Parallel do end end - defp new_state(opts) do - %{max_jobs: opts[:max_parallel] || 64, + defp new_state() do + %{max_jobs: Hex.State.fetch!(:http_concurrency), running: %{}, finished: %{}, waiting: :queue.new, diff --git a/lib/hex/registry.ex b/lib/hex/registry.ex index 41e709d9d..9177e1d23 100644 --- a/lib/hex/registry.ex +++ b/lib/hex/registry.ex @@ -1,28 +1,28 @@ defmodule Hex.Registry do @pdict_id :"$hex_registry" - @callback open(Keyword.t) :: {:ok, term} | {:error, String.t} - @callback close(term) :: boolean - @callback version(term) :: String.t - @callback installs(term) :: [{String.t, String.t}] - @callback stat(term) :: {non_neg_integer, non_neg_integer} - @callback search(term, String.t) :: [String.t] - @callback all_packages(term) :: [String.t] - @callback get_versions(term, String.t) :: [String.t] - @callback get_deps(term, String.t, String.t) :: [{String.t, String.t, String.t, boolean}] - @callback get_checksum(term, String.t, String.t) :: binary - @callback get_build_tools(term, String.t, String.t) :: [String.t] + @type name :: term + @type package :: String.t + @type version :: String.t + @type etag :: String.t + @callback open(Keyword.t) :: {:ok, name} | {:already_open, name} | {:error, String.t} + @callback close(name) :: :ok + @callback prefetch(name, [package]) :: :ok + @callback versions(name, package) :: [version] + @callback deps(name, package, version) :: [{String.t, String.t, String.t, boolean}] + @callback checksum(name, package, version) :: binary + @callback retired(name, package, version) :: binary + @callback tarball_etag(name, package, version) :: binary | nil + @callback tarball_etag(name, package, version, String.t) :: binary | nil options = quote do [ - version(), - installs(), - stat(), - search(term), - all_packages(), - get_versions(package), - get_deps(package, version), - get_checksum(package, version), - get_build_tools(package, version)] + prefetch(packages), + versions(package), + deps(package, version), + checksum(package, version), + retired(package, version), + tarball_etag(package, version), + tarball_etag(package, version, etag)] end Enum.each(options, fn {function, _, args} -> @@ -59,67 +59,10 @@ defmodule Hex.Registry do def close do case pdict_get() do {module, name} -> - result = module.close(name) + module.close(name) pdict_clean() - result nil -> false end end - - def info_installs do - installs = installs() - if version = latest_version(installs) do - Hex.Shell.warn "A new Hex version is available (#{version}), " <> - "please update with `mix local.hex`" - else - check_elixir_version(installs) - end - end - - defp latest_version(versions) do - current_elixir = System.version - current_hex = Hex.version - - versions - |> Enum.filter(fn {hex, _} -> Hex.Version.compare(hex, current_hex) == :gt end) - |> Enum.filter(fn {_, elixirs} -> Hex.Version.compare(hd(elixirs), current_elixir) != :gt end) - |> Enum.map(&elem(&1, 0)) - |> Enum.sort(&(Hex.Version.compare(&1, &2) == :gt)) - |> List.first - end - - defp check_elixir_version(versions) do - {:ok, built} = Hex.Version.parse(Hex.elixir_version()) - {:ok, current} = Hex.Version.parse(System.version) - - unless match_minor?(current, built) do - case :lists.keyfind(Hex.version, 1, versions) do - {_, elixirs} -> - if match_elixir_version?(elixirs, current) do - Hex.Shell.warn "Hex was built against Elixir #{Hex.elixir_version} " <> - "and you are running #{System.version}, please run `mix local.hex` " <> - "to update to a matching version" - end - - false -> - :ok - end - end - end - - defp match_elixir_version?(elixirs, current) do - Enum.any?(elixirs, fn elixir -> - {:ok, elixir} = Hex.Version.parse(elixir) - elixir.major == current.major and elixir.minor == current.minor - end) - end - - defp match_minor?(current, %Version{major: major, minor: minor}) do - lower = %Version{major: major, minor: minor, patch: 0, pre: [], build: nil} - upper = %Version{major: major, minor: minor + 1, patch: 0, pre: [0], build: nil} - - Hex.Version.compare(current, lower) in [:gt, :eq] and - Hex.Version.compare(current, upper) == :lt - end end diff --git a/lib/hex/registry/ets.ex b/lib/hex/registry/ets.ex deleted file mode 100644 index bbd70c677..000000000 --- a/lib/hex/registry/ets.ex +++ /dev/null @@ -1,166 +0,0 @@ -defmodule Hex.Registry.ETS do - @behaviour Hex.Registry - - @name __MODULE__ - @versions [3, 4] - @filename "registry.ets" - @timeout 60_000 - - def start_link do - Agent.start_link(fn -> nil end, name: @name) - end - - def open(opts) do - Agent.get_and_update(@name, fn - nil -> - path = opts[:registry_path] || path() - case :ets.file2tab(String.to_char_list(path)) do - {:ok, tid} -> - check_version(tid) - {{:ok, tid}, tid} - - {:error, reason} -> - if File.exists?(path) do - {{:error, inspect(reason)}, nil} - else - {{:error, "file does not exist, run `mix hex.info` to fetch it"}, nil} - end - end - - tid -> - {{:already_open, tid}, tid} - end, @timeout) - end - - def close do - if tid = Agent.get(@name, & &1) do - close(tid) - else - false - end - end - - def close(tid) do - Agent.get_and_update(@name, fn - nil -> - {false, nil} - agent_tid -> - ^agent_tid = tid - :ets.delete(tid) - {true, nil} - end, @timeout) - end - - def memory do - tid = Agent.get(@name, & &1) - :ets.info(tid, :memory) * :erlang.system_info(:wordsize) - end - - def path do - Path.join(Hex.State.fetch!(:home), @filename) - end - - def version(tid) do - case :ets.lookup(tid, :"$$version$$") do - [{:"$$version$$", version}] -> - version - _ -> - nil - end - end - - def installs(tid) do - case :ets.lookup(tid, :"$$installs2$$") do - [{:"$$installs2$$", installs}] -> - installs - _ -> - [] - end - end - - def stat(tid) do - fun = fn - {{package, version}, _}, {packages, releases} - when is_binary(package) and is_binary(version) -> - {packages, releases + 1} - {package, _}, {packages, releases} when is_binary(package) -> - {packages + 1, releases} - _, acc -> - acc - end - - :ets.foldl(fun, {0, 0}, tid) - end - - def search(tid, term) do - fun = fn - {package, list}, packages when is_binary(package) and is_list(list) -> - if String.contains?(package, term) do - [package|packages] - else - packages - end - _, packages -> - packages - end - - :ets.foldl(fun, [], tid) - |> Enum.sort - end - - def all_packages(tid) do - fun = fn - {package, list}, packages when is_binary(package) and is_list(list) -> - [package|packages] - _, packages -> - packages - end - - :ets.foldl(fun, [], tid) - |> Enum.sort - end - - def get_versions(tid, package) do - case :ets.lookup(tid, package) do - [] -> nil - [{^package, [versions|_]}] when is_list(versions) -> versions - end - end - - def get_deps(tid, package, version) do - case :ets.lookup(tid, {package, version}) do - [] -> - nil - [{{^package, ^version}, [deps|_]}] when is_list(deps) -> - Enum.map(deps, fn - [name, req, optional, app | _] -> {name, app, req, optional} - end) - end - end - - def get_checksum(tid, package, version) do - case :ets.lookup(tid, {package, version}) do - [] -> - nil - [{{^package, ^version}, [_, checksum | _]}] when is_nil(checksum) or is_binary(checksum) -> - checksum - end - end - - def get_build_tools(tid, package, version) do - case :ets.lookup(tid, {package, version}) do - [] -> - nil - [{{^package, ^version}, [_, _, build_tools | _]}] when is_list(build_tools) -> - build_tools - end - end - - defp check_version(tid) do - unless version(tid) in @versions do - raise Mix.Error, - message: "The registry file version is not supported. " <> - "Try updating Hex with `mix local.hex`." - end - end -end diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex new file mode 100644 index 000000000..c3f014535 --- /dev/null +++ b/lib/hex/registry/server.ex @@ -0,0 +1,409 @@ +defmodule Hex.Registry.Server do + use GenServer + @behaviour Hex.Registry + + # TODO: Optimize to not go through genserver + + @name __MODULE__ + @filename "cache.ets" + @timeout 60_000 + @update_interval 24 * 60 * 60 + + def start_link(opts \\ []) do + name = Keyword.get(opts, :name, @name) + opts = if name, do: [name: name], else: [] + GenServer.start_link(__MODULE__, [], opts) + end + + def open(name \\ @name, opts) do + GenServer.call(name, {:open, opts}, @timeout) + end + + def close do + GenServer.call(@name, :close, @timeout) + |> print_update_message + end + + def close(name) do + GenServer.call(name, :close, @timeout) + |> print_update_message + end + + def persist do + GenServer.call(@name, :persist, @timeout) + |> print_update_message + end + + def check_update do + GenServer.cast(@name, :check_update) + end + + def prefetch(name, packages) do + case GenServer.call(name, {:prefetch, packages}, @timeout) do + :ok -> + :ok + {:error, package} -> + Mix.raise "Hex is running in offline mode and the registry entry for " <> + "package #{package} is not cached locally" + end + end + + def versions(name, package) do + GenServer.call(name, {:versions, package}, @timeout) + end + + def deps(name, package, version) do + GenServer.call(name, {:deps, package, version}, @timeout) + end + + def checksum(name, package, version) do + GenServer.call(name, {:checksum, package, version}, @timeout) + end + + def retired(name, package, version) do + GenServer.call(name, {:retired, package, version}, @timeout) + end + + def tarball_etag(name, package, version) do + GenServer.call(name, {:tarball_etag, package, version}, @timeout) + end + + def tarball_etag(name, package, version, etag) do + GenServer.call(name, {:tarball_etag, package, version, etag}, @timeout) + end + + defp print_update_message({:update, {:http_error, reason}}) do + Hex.Shell.error "Hex update check failed, HTTP ERROR: #{inspect reason}" + :ok + end + defp print_update_message({:update, {:status, status}}) do + Hex.Shell.error "Hex update check failed, status code: #{status}" + :ok + end + defp print_update_message({:update, version}) do + Hex.Shell.warn "A new Hex version is available (#{Hex.version} < #{version}), " <> + "please update with `mix local.hex`" + :ok + end + defp print_update_message(:ok), do: :ok + + def init([]) do + {:ok, reset_state(%{})} + end + + defp reset_state(state) do + %{ets: nil, + path: nil, + pending: Hex.Set.new, + fetched: Hex.Set.new, + waiting: %{}, + waiting_close: nil, + already_checked_update?: Map.get(state, :already_checked_update?, false), + checking_update?: false, + new_update: nil} + end + + def handle_cast(:check_update, state) do + state = check_update(state, force: true) + {:noreply, state} + end + + def handle_call({:open, opts}, _from, %{ets: nil} = state) do + path = Hex.string_to_charlist(opts[:registry_path] || path()) + tid = open_ets(path) + state = %{state | ets: tid, path: path} + state = check_update(state, force: false) + {:reply, {:ok, self()}, state} + end + def handle_call({:open, _opts}, _from, state) do + {:reply, {:already_open, self()}, state} + end + + def handle_call(:close, from, state) do + maybe_wait_closing(state, from, fn + %{ets: nil} = state -> + state + %{ets: tid, path: path} -> + persist(tid, path) + :ets.delete(tid) + reset_state(state) + end) + end + + def handle_call(:persist, from, state) do + maybe_wait_closing(state, from, fn %{ets: tid, path: path} = state -> + persist(tid, path) + state + end) + end + + def handle_call({:prefetch, packages}, _from, state) do + packages = + packages + |> Enum.uniq + |> Enum.reject(&(&1 in state.fetched)) + |> Enum.reject(&(&1 in state.pending)) + + if Hex.State.fetch!(:offline?) do + prefetch_offline(packages, state) + else + prefetch_online(packages, state) + end + end + + def handle_call({:versions, package}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:versions, package}) + end) + end + + def handle_call({:deps, package, version}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:deps, package, version}) + end) + end + + def handle_call({:checksum, package, version}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:checksum, package, version}) + end) + end + + def handle_call({:retired, package, version}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:retired, package, version}) + end) + end + + def handle_call({:tarball_etag, package, version}, _from, state) do + etag = lookup(state.ets, {:tarball_etag, package, version}) + {:reply, etag, state} + end + + def handle_call({:tarball_etag, package, version, etag}, _from, state) do + :ets.insert(state.ets, {{:tarball_etag, package, version}, etag}) + {:reply, :ok, state} + end + + def handle_info({:DOWN, _ref, :process, _pid, :normal}, state) do + {:noreply, state} + end + + def handle_info({_ref, {:get_installs, result}}, state) do + result = + case result do + {code, body, _headers} when code in 200..299 -> + Hex.API.Registry.find_new_version_from_csv(body) + {code, body, _} -> + Hex.Shell.error("Failed to check for new Hex version") + Hex.Utils.print_error_result(code, body) + nil + end + + :ets.insert(state.ets, {:last_update, :calendar.universal_time}) + state = reply_to_update_waiting(state, result) + state = %{state | checking_update?: false} + {:noreply, state} + end + + def handle_info({:get_package, package, result}, state) do + pending = Hex.Set.delete(state.pending, package) + fetched = Hex.Set.put(state.fetched, package) + {replys, waiting} = Map.pop(state.waiting, package, []) + + write_result(result, package, state) + + Enum.each(replys, fn {from, fun} -> + GenServer.reply(from, fun.()) + end) + + state = %{state | pending: pending, waiting: waiting, fetched: fetched} + {:noreply, state} + end + + defp open_ets(path) do + case :ets.file2tab(path) do + {:ok, tid} -> + tid + {:error, {:read_error, {:file_error, _path, :enoent}}} -> + :ets.new(@name, []) + {:error, reason} -> + Hex.Shell.error("Error opening ETS file #{path}: #{inspect reason}") + File.rm(path) + :ets.new(@name, []) + end + end + + defp persist(tid, path) do + dir = Path.dirname(path) + File.mkdir_p!(dir) + :ok = :ets.tab2file(tid, path) + end + + defp prefetch_online(packages, state) do + Enum.each(packages, fn package -> + opts = fetch_opts(package, state) + Hex.Parallel.run(:hex_fetcher, {:registry, package}, [await: false], fn -> + {:get_package, package, Hex.API.Registry.get_package(package, opts)} + end) + end) + + pending = Enum.into(packages, state.pending) + state = %{state | pending: pending} + {:reply, :ok, state} + end + + defp prefetch_offline(packages, state) do + missing = + Enum.find(packages, fn package -> + unless lookup(state.ets, {:versions, package}), do: package + end) + + if missing do + {:reply, {:error, missing}, state} + else + fetched = Enum.into(packages, state.fetched) + {:reply, :ok, %{state | fetched: fetched}} + end + end + + defp write_result({code, body, headers}, package, %{ets: tid}) when code in 200..299 do + releases = + body + |> :zlib.gunzip + |> Hex.API.Registry.verify + |> Hex.API.Registry.decode + + delete_package(package, tid) + + Enum.each(releases, fn %{version: version, checksum: checksum, dependencies: deps} = release -> + :ets.insert(tid, {{:checksum, package, version}, checksum}) + :ets.insert(tid, {{:retired, package, version}, release[:retired]}) + deps = Enum.map(deps, fn dep -> + {dep[:package], dep[:app] || dep[:package], dep[:requirement], !!dep[:optional]} + end) + :ets.insert(tid, {{:deps, package, version}, deps}) + end) + + versions = Enum.map(releases, & &1[:version]) + :ets.insert(tid, {{:versions, package}, versions}) + + if etag = headers['etag'] do + :ets.insert(tid, {{:registry_etag, package}, List.to_string(etag)}) + end + end + defp write_result({304, _, _}, _package, _state) do + :ok + end + defp write_result({404, _, _}, package, %{ets: tid}) do + delete_package(package, tid) + :ok + end + + defp write_result({code, body, _}, package, %{ets: tid}) do + cached? = !!:ets.lookup(tid, {:versions, package}) + cached_message = if cached?, do: " (using cache)" + Hex.Shell.error("Failed to fetch record for '#{package}' from registry#{cached_message}") + Hex.Utils.print_error_result(code, body) + + unless cached? do + raise "Stopping due to errors" + end + end + + defp maybe_wait(package, from, state, fun) do + cond do + package in state.fetched -> + {:reply, fun.(), state} + package in state.pending -> + tuple = {from, fun} + waiting = Map.update(state.waiting, package, [tuple], &[tuple|&1]) + state = %{state | waiting: waiting} + {:noreply, state} + true -> + raise "Package #{package} not prefetched, please report this issue" + end + end + + defp fetch_opts(package, %{ets: tid}) do + case :ets.lookup(tid, {:registry_etag, package}) do + [{_, etag}] -> [etag: etag] + [] -> [] + end + end + + defp path do + Path.join(Hex.State.fetch!(:home), @filename) + end + + defp delete_package(package, tid) do + :ets.delete(tid, {:registry_etag, package}) + versions = lookup(tid, {:versions, package}) || [] + :ets.delete(tid, {:versions, package}) + Enum.each(versions, fn version -> + :ets.delete(tid, {:checksum, package, version}) + :ets.delete(tid, {:retired, package, version}) + :ets.delete(tid, {:deps, package, version}) + end) + end + + defp lookup(tid, key) do + case :ets.lookup(tid, key) do + [{^key, element}] -> element + [] -> nil + end + end + + def maybe_wait_closing(%{checking_update?: true, new_update: nil} = state, from, fun) do + state = %{state | waiting_close: {from, fun}} + {:noreply, state} + end + def maybe_wait_closing(%{checking_update?: false, new_update: nil} = state, _from, fun) do + {:reply, :ok, fun.(state)} + end + def maybe_wait_closing(%{checking_update?: false, new_update: new_update} = state, _from, fun) do + state = %{state | new_update: nil} + {:reply, {:update, new_update}, fun.(state)} + end + + defp reply_to_update_waiting(state, new_update) do + case state.waiting_close do + {from, fun} -> + reply = if new_update, do: {:update, new_update}, else: :ok + state = fun.(state) + GenServer.reply(from, reply) + %{state | waiting_close: nil} + nil -> + %{state | new_update: new_update} + end + end + + defp check_update(%{already_checked_update?: true} = state, _opts) do + state + end + defp check_update(%{checking_update?: true} = state, _opts) do + state + end + defp check_update(%{ets: tid} = state, opts) do + if opts[:force] || check_update?(tid) do + Task.async(fn -> + {:get_installs, Hex.API.Registry.get_installs} + end) + + %{state | checking_update?: true, already_checked_update?: true} + else + state + end + end + + defp check_update?(tid) do + if last = lookup(tid, :last_update) do + now = :calendar.universal_time |> :calendar.datetime_to_gregorian_seconds + last = :calendar.datetime_to_gregorian_seconds(last) + + now - last > @update_interval + else + true + end + end +end diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index dcad6bf5a..b5d7d46c7 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -5,46 +5,68 @@ defmodule Hex.RemoteConverger do alias Hex.Registry + def post_converge do + Registry.open!(Registry.Server) + :ok + after + Registry.pdict_clean + end + def remote?(dep) do !!dep.opts[:hex] end def converge(deps, lock) do Hex.start - Hex.Utils.ensure_registry!() - Hex.Registry.open!(Hex.Registry.ETS) - verify_lock(lock) + Registry.open!(Registry.Server) # We cannot use given lock here, because all deps that are being # converged have been removed from the lock by Mix # We need the old lock to get the children of Hex packages old_lock = Mix.Dep.Lock.read - locked = prepare_locked(lock, old_lock, deps) top_level = Hex.Mix.top_level(deps) flat_deps = Hex.Mix.flatten_deps(deps, top_level) - reqs = Hex.Mix.deps_to_requests(flat_deps) + requests = Hex.Mix.deps_to_requests(flat_deps) + + [Hex.Mix.packages_from_lock(lock), Hex.Mix.packages_from_lock(old_lock), Enum.map(requests, &elem(&1, 0))] + |> Enum.concat + |> Registry.prefetch + locked = prepare_locked(lock, old_lock, deps) + + check_lock(lock) check_deps(deps, top_level) - check_input(reqs, locked) + check_input(requests, locked) deps = Hex.Mix.prepare_deps(deps) top_level = Enum.map(top_level, &Atom.to_string/1) - Hex.Shell.info "Running dependency resolution" + Hex.Shell.info "Running dependency resolution..." - case Hex.Resolver.resolve(reqs, deps, top_level, locked) do + case Hex.Resolver.resolve(requests, deps, top_level, locked) do {:ok, resolved} -> print_success(resolved, locked) verify_resolved(resolved, old_lock) new_lock = Hex.Mix.to_lock(resolved) Hex.SCM.prefetch(new_lock) - Map.merge(lock, new_lock) + lock_merge(lock, new_lock) {:error, message} -> resolver_failed(message) end after - Hex.Registry.pdict_clean + if Version.compare(System.version, "1.4.0") == :lt do + Hex.Registry.Server.persist + end + end + + defp lock_merge(old, new) do + Map.merge(old, new, fn + _, {:hex, name, version, checksum, managers, deps}, {:hex, name, version, checksum, _managers, _deps} -> + {:hex, name, version, checksum, managers, deps} + _, _old, new -> + new + end) end defp resolver_failed(message) do @@ -60,7 +82,8 @@ defmodule Hex.RemoteConverger do def deps(%Mix.Dep{app: app}, lock) do case Hex.Utils.lock(lock[app]) do [:hex, name, version, _checksum, _managers, nil] -> - Hex.Utils.ensure_registry!(fetch: false) + Registry.open!(Registry.Server) + Registry.prefetch([Atom.to_string(name)]) get_deps(name, version) [:hex, _name, _version, _checksum, _managers, deps] -> deps @@ -68,12 +91,12 @@ defmodule Hex.RemoteConverger do [] end after - Hex.Registry.pdict_clean + Registry.pdict_clean end defp get_deps(name, version) do name = Atom.to_string(name) - deps = try_get_deps(name, version) + deps = Registry.deps(name, version) || [] for {name, app, req, optional} <- deps do app = String.to_atom(app) @@ -88,15 +111,6 @@ defmodule Hex.RemoteConverger do end end - defp try_get_deps(name, version) do - if deps = Registry.get_deps(name, version) do - deps - else - Hex.Utils.ensure_registry!() - Registry.get_deps(name, version) || [] - end - end - defp check_deps(deps, top_level) do Enum.each(deps, fn dep -> if dep.app in top_level and dep.scm == Hex.SCM and is_nil(dep.requirement) do @@ -106,8 +120,8 @@ defmodule Hex.RemoteConverger do end) end - defp check_input(reqs, locked) do - Enum.each(reqs, fn {name, _app, req, from} -> + defp check_input(requests, locked) do + Enum.each(requests, fn {name, _app, req, from} -> check_package_req(name, req, from) end) @@ -117,7 +131,7 @@ defmodule Hex.RemoteConverger do end defp check_package_req(name, req, from) do - if Registry.get_versions(name) do + if Registry.versions(name) do if req != nil and Hex.Version.parse_requirement(req) == :error do Mix.raise "Required version #{inspect req} for package #{name} is incorrectly specified (from: #{from})" end @@ -132,53 +146,78 @@ defmodule Hex.RemoteConverger do resolved = Map.drop(resolved, locked) if Map.size(resolved) != 0 do - Hex.Shell.info "Dependency resolution completed" + Hex.Shell.info "Dependency resolution completed:" resolved = Enum.sort(resolved) Enum.each(resolved, fn {name, version} -> - Hex.Shell.info " #{name}: #{version}" + name + |> Registry.retired(version) + |> print_status(name, version) end) end end + defp print_status(nil, name, version) do + Hex.Shell.info IO.ANSI.format [:green, " #{name} #{version}"] + end + + defp print_status(retired, name, version) do + Hex.Shell.warn " #{name} #{version} RETIRED!" + Hex.Shell.warn " (#{retirement_reason(retired[:reason])}) #{retired[:message]}" + end + + defp retirement_reason(:RETIRED_OTHER), do: "other" + defp retirement_reason(:RETIRED_INVALID), do: "invalid" + defp retirement_reason(:RETIRED_SECURITY), do: "security" + defp retirement_reason(:RETIRED_DEPRECATED), do: "deprecated" + defp retirement_reason(:RETIRED_RENAMED), do: "renamed" + defp retirement_reason(other), do: other + defp verify_resolved(resolved, lock) do Enum.each(resolved, fn {name, app, version} -> atom_name = String.to_atom(name) case Hex.Utils.lock(lock[String.to_atom(app)]) do [:hex, ^atom_name, ^version, checksum, _managers, deps] -> - registry_checksum = Hex.Registry.get_checksum(name, version) - - if checksum && Base.decode16!(checksum, case: :lower) != Base.decode16!(registry_checksum), - do: Mix.raise "Registry checksum mismatch against lock (#{name} #{version})" - - if deps do - deps = - Enum.map(deps, fn {app, req, opts} -> - {Atom.to_string(opts[:hex]), Atom.to_string(app), req, !!opts[:optional]} - end) - - if Enum.sort(deps) != Enum.sort(Hex.Registry.get_deps(name, version)), - do: Mix.raise "Registry dependencies mismatch against lock (#{name} #{version})" - end + verify_registry(deps, name, version, checksum) _ -> :ok end end) end - defp verify_lock(lock) do + defp verify_registry(deps, name, version, checksum) do + registry_checksum = Registry.checksum(name, version) + if checksum && Base.decode16!(checksum, case: :lower) != registry_checksum do + Mix.raise "Registry checksum mismatch against lock (#{name} #{version})" + end + + if deps, do: verify_dependencies(deps, name, version) + end + + defp verify_dependencies(deps, name, version) do + deps = + Enum.map(deps, fn {app, req, opts} -> + {Atom.to_string(opts[:hex]), Atom.to_string(app), req, !!opts[:optional]} + end) + + if Enum.sort(deps) != Enum.sort(Registry.deps(name, version)) do + Mix.raise "Registry dependencies mismatch against lock (#{name} #{version})" + end + end + + defp check_lock(lock) do Enum.each(lock, fn {_app, info} -> case Hex.Utils.lock(info) do [:hex, name, version, _checksum, _managers, _deps] -> - verify_dep(Atom.to_string(name), version) + check_dep(Atom.to_string(name), version) _ -> :ok end end) end - defp verify_dep(app, version) do - if versions = Registry.get_versions(app) do + defp check_dep(app, version) do + if versions = Registry.versions(app) do unless version in versions do Mix.raise "Unknown package version #{app} #{version} in lockfile" end @@ -198,8 +237,8 @@ defmodule Hex.RemoteConverger do [:hex, name, version, _checksum, _managers, nil] -> # Do not error on bad data in the old lock because we should just # fix it automatically - if deps = Registry.get_deps(Atom.to_string(name), version) do - apps = Enum.map(deps, &elem(&1, 0)) + if deps = Registry.deps(Atom.to_string(name), version) do + apps = Enum.map(deps, &elem(&1, 1)) [apps, do_with_children(apps, lock)] else [] diff --git a/lib/hex/repo.ex b/lib/hex/repo.ex index f940aad14..e216c3e30 100644 --- a/lib/hex/repo.ex +++ b/lib/hex/repo.ex @@ -6,14 +6,17 @@ defmodule Hex.Repo do def request(url, etag) do opts = [body_format: :binary] headers = [{'user-agent', Hex.API.user_agent}] - headers = if etag, do: [{'if-none-match', '"' ++ etag ++ '"'}|headers], else: headers + headers = if etag, do: [{'if-none-match', Hex.string_to_charlist(etag)}|headers], else: headers http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) - url = String.to_char_list(url) + url = Hex.string_to_charlist(url) profile = Hex.State.fetch!(:httpc_profile) case Hex.API.request_with_redirect(:get, {url, headers}, http_opts, opts, profile, 3) do - {:ok, {{_version, 200, _reason}, _headers, body}} -> - {:ok, body} + {:ok, {{_version, 200, _reason}, headers, body}} -> + headers = Enum.into(headers, %{}) + etag = headers['etag'] + etag = if etag, do: List.to_string(etag) + {:ok, body, etag} {:ok, {{_version, 304, _reason}, _headers, _body}} -> {:ok, :cached} {:ok, {{_version, code, _reason}, _headers, _body}} -> diff --git a/lib/hex/resolver.ex b/lib/hex/resolver.ex index 09ea582c3..815afd0cf 100644 --- a/lib/hex/resolver.ex +++ b/lib/hex/resolver.ex @@ -86,7 +86,7 @@ defmodule Hex.Resolver do defp activate(request(app: app, name: name), pending, [version|versions], optional, info, activated, parents) do {new_pending, new_optional, new_deps} = get_deps(app, name, version, info, activated) - new_pending = new_pending ++ pending + new_pending = pending ++ new_pending new_optional = merge_optional(optional, new_optional) state = state(activated: activated, requests: pending, optional: optional, deps: info(info, :deps)) @@ -134,7 +134,7 @@ defmodule Hex.Resolver do end defp get_versions(package, requests) do - if versions = Registry.get_versions(package) do + if versions = Registry.versions(package) do Enum.reduce(requests, versions, fn request, versions -> req = request(request, :req) Enum.filter(versions, &version_match?(&1, req)) @@ -146,7 +146,9 @@ defmodule Hex.Resolver do end defp get_deps(app, package, version, info(top_level: top_level, deps: all_deps), activated) do - if deps = Registry.get_deps(package, version) do + if deps = Registry.deps(package, version) do + dep_names = Enum.map(deps, &elem(&1, 0)) + Registry.prefetch(dep_names) all_deps = attach_dep_and_children(all_deps, app, deps) overridden_map = overridden_parents(top_level, all_deps, app) @@ -195,9 +197,9 @@ defmodule Hex.Resolver do # ugly hack to check if we should print the pre-release explanation if message =~ "*\n" do message <> - "\n* This requirement failed because by default pre-releases " <> - "are never matched. To match against pre-releases include " <> - "a pre-release in the requirement string: \"~> 2.0-beta\".\n" + "\n* This requirement does not match pre-releases. " <> + "To match pre-releases include a pre-release in the " <> + "requirement, such as: \"~> 2.0-beta\".\n" else message end diff --git a/lib/hex/resolver/backtracks.ex b/lib/hex/resolver/backtracks.ex index 25e161c5e..66e7c2378 100644 --- a/lib/hex/resolver/backtracks.ex +++ b/lib/hex/resolver/backtracks.ex @@ -99,7 +99,6 @@ defmodule Hex.Resolver.Backtracks do :error -> Map.put(map, name, [{[version], parents}|list]) end - :error -> Map.put(map, name, [{[version], parents}]) end @@ -186,13 +185,11 @@ defmodule Hex.Resolver.Backtracks do end defp parent_message({parent(name: "mix.lock", version: [], requirement: req), {color, pre_failed?}}) do - ["Locked to ", color, requirement(req), :reset, " in your mix.lock", - pre_message(pre_failed?)] + [:bright, "mix.lock", :reset, " specifies ", color, requirement(req), :reset, pre_message(pre_failed?)] end defp parent_message({parent(name: "mix.exs", version: [], requirement: req), {color, pre_failed?}}) do - ["You specified ", color, requirement(req), :reset, " in your mix.exs", - pre_message(pre_failed?)] + [:bright, "mix.exs", :reset, " specifies ", color, requirement(req), :reset, pre_message(pre_failed?)] end defp parent_message({parent(name: name, version: versions, requirement: req), {color, pre_failed?}}) do @@ -228,7 +225,7 @@ defmodule Hex.Resolver.Backtracks do defp parent_reason(nil, _child, _versions), do: nil defp parent_reason(parent, child, []) do - versions = Hex.Registry.get_versions(child) + versions = Hex.Registry.versions(child) parent_reason(parent, child, versions) end defp parent_reason(parent(requirement: req), _child, versions) do @@ -263,8 +260,14 @@ defmodule Hex.Resolver.Backtracks do {[x, y], _} -> [" (versions ", to_string(x), " and ", to_string(y), ")"] {_, true} when length(versions) > 2 -> - first = versions |> List.first |> to_string - last = versions |> List.last |> to_string + first = + versions + |> List.first + |> to_string + last = + versions + |> List.last + |> to_string [" (versions ", first, " to ", last, ")"] _ -> versions = Enum.map(versions, &to_string/1) @@ -274,7 +277,7 @@ defmodule Hex.Resolver.Backtracks do defp merge_versions?(_package, []), do: false defp merge_versions?(package, versions) do - all_versions = Hex.Registry.get_versions(package) + all_versions = Hex.Registry.versions(package) sub_range?(all_versions, versions) end diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 0a68bdd3b..8476a3422 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -48,6 +48,7 @@ defmodule Hex.SCM do case File.read(Path.join(dest, ".hex")) do {:ok, file} -> case parse_manifest(file) do + {^name, ^version, ^checksum, _} -> :ok {^name, ^version, ^checksum} -> :ok {^name, ^version, _} when is_nil(checksum) -> :ok {^name, ^version} -> :ok @@ -65,25 +66,18 @@ defmodule Hex.SCM do def managers(opts) do case Hex.Utils.lock(opts[:lock]) do - [:hex, name, version, _checksum, nil, _deps] -> - Hex.Utils.ensure_registry!(fetch: false) - name = Atom.to_string(name) - build_tools = Hex.Registry.get_build_tools(name, version) || [] - Enum.map(build_tools, &String.to_atom/1) [:hex, _name, _version, _checksum, managers, _deps] -> - managers + managers || [] _ -> [] end - after - Hex.Registry.pdict_clean end def checkout(opts) do - Hex.Registry.open!(Hex.Registry.ETS) + Hex.Registry.open!(Hex.Registry.Server) lock = Hex.Utils.lock(opts[:lock]) |> ensure_lock(opts) - [:hex, _name, version, checksum, _managers, _deps] = lock + [:hex, lock_name, version, checksum, _managers, deps] = lock name = opts[:hex] dest = opts[:dest] @@ -93,12 +87,15 @@ defmodule Hex.SCM do Hex.Shell.info " Checking package (#{url})" - case Hex.Parallel.await(:hex_fetcher, {name, version}, @fetch_timeout) do + case Hex.Parallel.await(:hex_fetcher, {:tarball, name, version}, @fetch_timeout) do {:ok, :cached} -> Hex.Shell.info " Using locally cached package" {:ok, :offline} -> Hex.Shell.info " [OFFLINE] Using locally cached package" - {:ok, :new} -> + {:ok, :new, etag} -> + Hex.Registry.tarball_etag(name, version, etag) + if Version.compare(System.version, "1.4.0") == :lt, + do: Hex.Registry.Server.persist Hex.Shell.info " Fetched package" {:error, reason} -> Hex.Shell.error(reason) @@ -109,11 +106,18 @@ defmodule Hex.SCM do end File.rm_rf!(dest) - Hex.Tar.unpack(path, dest, {name, version}) - manifest = encode_manifest(name, version, checksum) + + meta = Hex.Tar.unpack(path, dest, {name, version}) + build_tools = guess_build_tools(meta) + managers = + build_tools + |> Enum.map(&String.to_atom/1) + |> Enum.sort + + manifest = encode_manifest(name, version, checksum, managers) File.write!(Path.join(dest, ".hex"), manifest) - opts[:lock] + {:hex, lock_name, version, checksum, managers, Enum.sort(deps)} after Hex.Registry.pdict_clean end @@ -122,6 +126,34 @@ defmodule Hex.SCM do checkout(opts) end + @build_tools [ + {"mix.exs" , "mix"}, + {"rebar.config", "rebar"}, + {"rebar" , "rebar"}, + {"Makefile" , "make"}, + {"Makefile.win", "make"} + ] + + def guess_build_tools(%{"build_tools" => tools}) do + if tools, + do: Enum.uniq(tools), + else: [] + end + + def guess_build_tools(meta) do + base_files = + (meta["files"] || []) + |> Enum.filter(&(Path.dirname(&1) == ".")) + |> Enum.into(Hex.Set.new) + + Enum.flat_map(@build_tools, fn {file, tool} -> + if file in base_files, + do: [tool], + else: [] + end) + |> Enum.uniq + end + defp ensure_lock(nil, opts) do Mix.raise "The lock is missing for package #{opts[:hex]}. This could be " <> "because another package has configured the application name " <> @@ -130,15 +162,29 @@ defmodule Hex.SCM do end defp ensure_lock(lock, _opts), do: lock - defp parse_manifest(file) do - file - |> String.strip - |> String.split(",") - |> List.to_tuple + def parse_manifest(file) do + lines = + file + |> Hex.string_trim + |> String.split("\n") + + case lines do + [first] -> + (String.split(first, ",") ++ [[]]) + |> List.to_tuple + [first, managers] -> + managers = + managers + |> String.split(",") + |> Enum.map(&String.to_atom/1) + (String.split(first, ",") ++ [managers]) + |> List.to_tuple + end end - defp encode_manifest(name, version, checksum) do - "#{name},#{version},#{checksum}" + defp encode_manifest(name, version, checksum, managers) do + managers = managers || [] + "#{name},#{version},#{checksum}\n#{Enum.join(managers, ",")}" end defp cache_path do @@ -152,11 +198,12 @@ defmodule Hex.SCM do def prefetch(lock) do fetch = fetch_from_lock(lock) - Enum.each(fetch, fn {name, version} -> - Hex.Parallel.run(:hex_fetcher, {name, version}, fn -> - filename = "#{name}-#{version}.tar" - path = cache_path(filename) - fetch(filename, path) + Enum.each(fetch, fn {package, version} -> + filename = "#{package}-#{version}.tar" + path = cache_path(filename) + etag = File.exists?(path) && Hex.Registry.tarball_etag(package, version) + Hex.Parallel.run(:hex_fetcher, {:tarball, package, version}, fn -> + fetch(filename, path, etag) end) end) end @@ -179,18 +226,17 @@ defmodule Hex.SCM do end) end - defp fetch(name, path) do + defp fetch(name, path, etag) do if Hex.State.fetch!(:offline?) do {:ok, :offline} else - etag = Hex.Utils.etag(path) - url = Hex.API.repo_url("tarballs/#{name}") + url = Hex.API.repo_url("tarballs/#{name}") File.mkdir_p!(cache_path()) case Hex.Repo.request(url, etag) do - {:ok, body} when is_binary(body) -> + {:ok, body, etag} -> File.write!(path, body) - {:ok, :new} + {:ok, :new, etag} other -> other end diff --git a/lib/hex/set.ex b/lib/hex/set.ex new file mode 100644 index 000000000..4af921e05 --- /dev/null +++ b/lib/hex/set.ex @@ -0,0 +1,15 @@ +defmodule Hex.Set do + if Version.compare(System.version, "1.1.0") == :lt do + @module HashSet + else + @module MapSet + end + + def new(enum) do + Enum.into(enum, new()) + end + + defdelegate new(), to: @module + defdelegate put(set, value), to: @module + defdelegate delete(set, value), to: @module +end diff --git a/lib/hex/state.ex b/lib/hex/state.ex index ea38af38f..ec951133d 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -42,12 +42,12 @@ defmodule Hex.State do https_proxy: load_config(config, ["https_proxy", "HTTPS_PROXY"], :https_proxy), offline?: load_config(config, ["HEX_OFFLINE"], :offline) |> to_boolean |> default(false), check_cert?: load_config(config, ["HEX_UNSAFE_HTTPS"], :unsafe_https) |> to_boolean |> default(false) |> Kernel.not, - check_registry?: load_config(config, ["HEX_UNSAFE_REGISTRY"], :unsafe_registry) |> to_boolean |> default(true), + check_registry?: load_config(config, ["HEX_UNSAFE_REGISTRY"], :unsafe_registry) |> to_boolean |> default(false) |> Kernel.not, + http_concurrency: load_config(config, ["HEX_HTTP_CONCURRENCY"], :http_concurrency) |> to_integer |> default(8), hexpm_pk: @hexpm_pk, - registry_updated: false, httpc_profile: :hex, ssl_version: ssl_version(), - pbkdf2_iters: 32768, + pbkdf2_iters: 32_768, clean_pass: true} end @@ -56,7 +56,10 @@ defmodule Hex.State do if Mix.env == :test do def fetch(:httpc_profile) do profile = make_ref() |> :erlang.ref_to_list |> List.to_atom - {:ok, _pid} = :httpc_manager.start_link(profile, :only_session_cookies, :stand_alone) + {:ok, pid} = :httpc_manager.start_link(profile, :only_session_cookies, :stand_alone) + # Unlink to avoid race conditions where the manager closes before all requests finished + Process.unlink(pid) + {:ok, pid} end end @@ -137,6 +140,13 @@ defmodule Hex.State do defp to_boolean("false"), do: false defp to_boolean("true"), do: true + defp to_integer(nil), do: nil + defp to_integer(""), do: nil + defp to_integer(string) do + {int, _} = Integer.parse(string) + int + end + defp default(nil, value), do: value defp default(value, _), do: value @@ -189,9 +199,4 @@ defmodule Hex.State do do: [major, minor, patch] defp version_pad([major, minor, patch | _]), do: [major, minor, patch] - - defp to_integer(string) do - {int, _} = Integer.parse(string) - int - end end diff --git a/lib/hex/tar.ex b/lib/hex/tar.ex index 179fbbc92..c5b6042e2 100644 --- a/lib/hex/tar.ex +++ b/lib/hex/tar.ex @@ -1,8 +1,7 @@ defmodule Hex.Tar do - @supported [nil, "2", "3"] + @supported ["3"] @version "3" - @required_files_2 ~w(VERSION CHECKSUM metadata.exs contents.tar.gz)c - @required_files_3 ~w(VERSION CHECKSUM metadata.config contents.tar.gz)c + @required_files ~w(VERSION CHECKSUM metadata.config contents.tar.gz)c def create(meta, files, cleanup_tarball? \\ true) do contents_path = "#{meta[:name]}-#{meta[:version]}-contents.tar.gz" @@ -10,8 +9,8 @@ defmodule Hex.Tar do files = Enum.map(files, fn - {name, bin} -> {String.to_char_list(name), bin} - name -> String.to_char_list(name) + {name, bin} -> {Hex.string_to_charlist(name), bin} + name -> Hex.string_to_charlist(name) end) :ok = :erl_tar.create(contents_path, files, [:compressed]) @@ -25,7 +24,7 @@ defmodule Hex.Tar do {'VERSION', @version}, {'CHECKSUM', checksum}, {'metadata.config', meta_string}, - {'contents.tar.gz', contents} ] + {'contents.tar.gz', contents}] :ok = :erl_tar.create(path, files) tar = File.read!(path) @@ -38,11 +37,11 @@ defmodule Hex.Tar do case :erl_tar.extract(path, [:memory]) do {:ok, files} -> files = Enum.into(files, %{}) - tar_version = files['VERSION'] - check_version(tar_version) - check_files(tar_version, files) - checksum(tar_version, files, {name, version}) + check_version(files['VERSION']) + check_files(files) + checksum(files, {name, version}) extract_contents(files['contents.tar.gz'], dest) + decode_metadata(files['metadata.config']) :ok -> Mix.raise "Unpacking tarball failed: tarball empty" @@ -52,17 +51,9 @@ defmodule Hex.Tar do end end - defp check_files(version, files) do + defp check_files(files) do files = Map.keys(files) - - cond do - version == "2" -> - diff_files(@required_files_2, files) - version == "3" -> - diff_files(@required_files_3, files) - true -> - :ok - end + diff_files(@required_files, files) end defp diff_files(required, given) do @@ -80,17 +71,17 @@ defmodule Hex.Tar do end end - defp checksum(tar_version, files, {name, version}) do + defp checksum(files, {name, version}) do case Base.decode16(files['CHECKSUM'], case: :mixed) do {:ok, tar_checksum} -> - meta = metadata(tar_version, files) + meta = files['metadata.config'] blob = files['VERSION'] <> meta <> files['contents.tar.gz'] - registry_checksum = Hex.Registry.get_checksum(to_string(name), version) + registry_checksum = Hex.Registry.checksum(to_string(name), version) checksum = :crypto.hash(:sha256, blob) if checksum != tar_checksum, do: Mix.raise "Checksum mismatch in tarball" - if checksum != Base.decode16!(registry_checksum), + if checksum != registry_checksum, do: Mix.raise "Checksum mismatch against registry" :error -> @@ -111,9 +102,6 @@ defmodule Hex.Tar do end end - defp metadata("2", files), do: files['metadata.exs'] - defp metadata("3", files), do: files['metadata.config'] - defp encode_term(list) do list |> Hex.Utils.binarify(maps: false) @@ -129,4 +117,33 @@ defmodule Hex.Tar do :erl_tar.format_error(reason) |> List.to_string end + + defp decode_metadata(contents) do + string = safe_to_charlist(contents) + case :safe_erl_term.string(string) do + {:ok, tokens, _line} -> + try do + terms = :safe_erl_term.terms(tokens) + Enum.into(terms, %{}) + rescue + FunctionClauseError -> + Mix.raise "Error reading package metadata: invalid terms" + ArgumentError -> + Mix.raise "Error reading package metadata: not in key-value format" + end + + {:error, reason} -> + Mix.raise "Error reading package metadata: #{inspect reason}" + end + end + + # Some older packages have invalid unicode + defp safe_to_charlist(string) do + try do + Hex.string_to_charlist(string) + rescue + UnicodeConversionError -> + :erlang.binary_to_list(string) + end + end end diff --git a/lib/hex/utils.ex b/lib/hex/utils.ex index 206cde617..62c6c9428 100644 --- a/lib/hex/utils.ex +++ b/lib/hex/utils.ex @@ -1,153 +1,4 @@ defmodule Hex.Utils do - @public_keys_html "https://hex.pm/docs/public_keys" - - def ensure_registry(opts \\ []) do - update_result = update_registry(opts) - - if update_result == :error do - if File.exists?(Hex.Registry.ETS.path) do - Hex.Shell.warn("Failed to update registry (using the cache)") - else - {:error, :update_failed} - end - else - start_result = - if Keyword.get(opts, :open, true), - do: Hex.Registry.open(Hex.Registry.ETS), - else: :nope - - # Show available newer versions - if update_result in [{:ok, :new}, {:ok, :no_fetch}] and start_result == :ok do - Hex.Registry.info_installs - end - - start_result - end - end - - def ensure_registry!(opts \\ []) do - update_result = update_registry(opts) - - if update_result == :error do - if File.exists?(Hex.Registry.ETS.path) do - Hex.Shell.warn("Failed to update registry (using the cache)") - else - Mix.raise "Failed to fetch registry" - end - end - - start_result = - if Keyword.get(opts, :open, true), - do: Hex.Registry.open!(Hex.Registry.ETS), - else: :nope - - # Show available newer versions - if update_result in [{:ok, :new}, {:ok, :no_fetch}] and start_result == :ok do - Hex.Registry.info_installs - end - end - - defp update_registry(opts) do - path = Hex.Registry.ETS.path - path_gz = path <> ".gz" - - cond do - Hex.State.fetch!(:offline?) -> - {:ok, :offline} - Hex.State.fetch!(:registry_updated) -> - {:ok, :cached} - not Keyword.get(opts, :fetch, true) -> - {:ok, :no_fetch} - true -> - Hex.State.put(:registry_updated, true) - closed? = Hex.Registry.close - - try do - api_opts = - if Keyword.get(opts, :cache, true) do - [etag: etag(path_gz)] - else - [] - end - - case Hex.API.Registry.get(api_opts) do - {200, body, headers} -> - Hex.State.fetch!(:check_registry?) && verify_registry!(body, headers) - File.mkdir_p!(Path.dirname(path)) - File.write!(path_gz, body) - data = :zlib.gunzip(body) - File.write!(path, data) - {:ok, :new} - {304, _, _} -> - {:ok, :new} - {code, body, _} -> - Hex.Shell.error "Registry update failed (#{code})" - print_error_result(code, body) - :error - end - after - # Open registry if it was already open when update began - if closed?, do: Hex.Registry.open!(Hex.Registry.ETS) - end - end - end - - defp verify_registry!(body, headers) do - domain = if repo = Hex.State.fetch!(:repo), do: repo, else: "hex.pm" - - signature = headers['x-hex-signature'] || - headers['x-amz-meta-signature'] || - get_signature(domain) - - signature = signature |> to_string |> Base.decode16!(case: :lower) - key = Hex.PublicKey.public_keys(domain) - - unless key do - Mix.raise "No public key stored for #{domain}. Either install a public " <> - "key with `mix hex.public_keys` or disable the registry " <> - "verification check by setting `HEX_UNSAFE_REGISTRY=1`." - end - - unless Hex.PublicKey.verify(body, :sha512, signature, [key]) do - Mix.raise "Could not verify authenticity of fetched registry file. " <> - "This may happen because a proxy or some entity is " <> - "interfering with the download or because you don't have a " <> - "public key to verify the registry.\n\nYou may try again " <> - "later or check if a new public key has been released in " <> - "our public keys page: #{@public_keys_html}" - end - end - - defp get_signature(domain) do - case Hex.API.Registry.get_signature do - {200, body, _} -> - body - other -> - reason = signature_fetch_fail(other) - Mix.raise "The repository at #{domain} did not provide a signature " <> - "for the registry because it #{reason}. This could be because " <> - "of a man-in-the-middle attack or simply because the repository " <> - "does not sign its registry. The signature verification check " <> - "can be disabled by setting `HEX_UNSAFE_REGISTRY=1`." - end - end - - defp signature_fetch_fail({:http_error, reason, _}), - do: "failed with http error #{inspect reason}" - defp signature_fetch_fail({code, _, _}), - do: "returned http status code #{code}" - - def etag(path) do - case File.read(path) do - {:ok, binary} -> - :crypto.hash(:md5, binary) - |> Base.encode16(case: :lower) - |> String.to_char_list - {:error, _} -> - nil - end - end - def safe_deserialize_erlang("") do nil end @@ -199,7 +50,7 @@ defmodule Hex.Utils do Hex.Shell.info body end - def print_error_result(status, body) do + def print_error_result(status, body) when is_map(body) do message = body["message"] errors = body["errors"] @@ -248,6 +99,11 @@ defmodule Hex.Utils do def hexdocs_url(package, version), do: "https://hexdocs.pm/#{package}/#{version}" + def hexdocs_module_url(package, module), + do: "https://hexdocs.pm/#{package}/#{module}.html" + def hexdocs_module_url(package, version, module), + do: "https://hexdocs.pm/#{package}/#{version}/#{module}.html" + def proxy_config(url) do {http_proxy, https_proxy} = proxy_setup() proxy_auth(URI.parse(url), http_proxy, https_proxy) @@ -263,7 +119,7 @@ defmodule Hex.Utils do uri = URI.parse(proxy) if uri.host && uri.port do - host = String.to_char_list(uri.host) + host = Hex.string_to_charlist(uri.host) :httpc.set_options([{proxy_scheme(scheme), {{host, uri.port}, []}}], :hex) end @@ -289,8 +145,8 @@ defmodule Hex.Utils do defp proxy_auth(%URI{userinfo: auth}) do destructure [user, pass], String.split(auth, ":", parts: 2) - user = String.to_char_list(user) - pass = String.to_char_list(pass || "") + user = Hex.string_to_charlist(user) + pass = Hex.string_to_charlist(pass || "") [proxy_auth: {user, pass}] end @@ -315,5 +171,9 @@ defmodule Hex.Utils do def lock(nil), do: nil def lock({:hex, name, version}), do: [:hex, name, version, nil, nil, nil] - def lock(tuple), do: tuple |> Tuple.to_list |> Enum.take(6) + def lock(tuple) do + tuple + |> Tuple.to_list + |> Enum.take(6) + end end diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 5edcad0b4..7226c8102 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -7,18 +7,13 @@ defmodule Mix.Hex.Build do @meta_fields @error_fields ++ @warn_fields ++ ~w(elixir extra)a @max_description_length 300 - def prepare_package! do - Hex.start - Hex.Utils.ensure_registry(fetch: false) + def prepare_package do Mix.Project.get! - config = Mix.Project.config raise_if_umbrella_project!(config) package = Enum.into(config[:package] || [], %{}) - {deps, exclude_deps} = dependencies(config) - meta = meta_for(config, package, deps) %{config: config, package: package, deps: deps, @@ -36,32 +31,29 @@ defmodule Mix.Hex.Build do end) end - Enum.each(@meta_fields, &print_meta(meta, &1)) + Enum.each(@meta_fields, &print_metadata(meta, &1)) errors = - error_missing!(meta) ++ - error_long_description(meta) ++ - error_missing_files(package_files) ++ + check_missing_fields(meta) ++ + check_description_length(meta) ++ + check_missing_files(package_files || []) ++ check_excluded_deps(exclude_deps) if errors != [] do - error_msg = - ["Stopping package build due to errors." | errors] - |> Enum.join("\n") - Mix.raise(error_msg) + ["Stopping package build due to errors." | errors] + |> Enum.join("\n") + |> Mix.raise() end end defp check_excluded_deps([]), do: [] defp check_excluded_deps(deps) do - deps - |> Enum.into(["Excluded dependencies (not part of the Hex package):"], &(" #{&1}")) - |> Enum.join("\n") - |> List.wrap() + ["Dependencies excluded from the package (only Hex packages can be dependencies): #{Enum.join(deps, ", ")}"] end defp meta_for(config, package, deps) do - Keyword.take(config, [:app, :version, :elixir, :description]) + config + |> Keyword.take([:app, :version, :elixir, :description]) |> Enum.into(%{}) |> Map.merge(package) |> package(config) @@ -70,7 +62,8 @@ defmodule Mix.Hex.Build do defp dependencies(meta) do deps = Enum.map(meta[:deps] || [], &Hex.Mix.dep/1) - {include, exclude} = Enum.partition(deps, &(package_dep?(&1) and prod_dep?(&1))) + deps = Enum.filter(deps, &prod_dep?/1) + {include, exclude} = Enum.partition(deps, &package_dep?/1) Enum.each(include, fn {app, _req, opts} -> if opts[:override] do @@ -100,15 +93,15 @@ defmodule Mix.Hex.Build do package |> Map.put(:files, files) - |> maybe_put(:description, fn _ -> package[:description] end, &String.strip/1) - |> maybe_put(:name, fn _ -> package[:name] || config[:app] end, & &1) - |> maybe_put(:build_tools, fn _ -> !package[:build_tools] && guess_build_tools(files) end, & &1) + |> maybe_put(:description, package[:description], &Hex.string_trim/1) + |> maybe_put(:name, package[:name] || config[:app], &(&1)) + |> maybe_put(:build_tools, !package[:build_tools] && guess_build_tools(files), &(&1)) |> Map.take(@meta_fields) end - defp maybe_put(map, key, check, value) do - if result = check.(map) do - Map.put(map, key, value.(result)) + defp maybe_put(map, key, value, transform) do + if value do + Map.put(map, key, transform.(value)) else map end @@ -151,72 +144,63 @@ defmodule Mix.Hex.Build do end end - defp print_meta(meta, :files) do - if meta[:files] != [] do - Hex.Shell.info(" Files:") - Enum.each(meta[:files], &Hex.Shell.info(" #{&1}")) - else - Hex.Shell.error("No files") + defp print_metadata(metadata, :files) do + case metadata[:files] do + [] -> + Hex.Shell.error("No files") + files -> + Hex.Shell.info(" Files:") + Enum.each(files, &Hex.Shell.info(" #{&1}")) end end - defp print_meta(meta, key) do - if value = meta[key] do - key = key |> Atom.to_string |> String.replace("_", " ") |> String.capitalize - value = meta_value(value) + defp print_metadata(metadata, key) do + if value = metadata[key] do + key = + key + |> Atom.to_string + |> String.replace("_", " ") + |> String.capitalize + value = format_metadata_value(value) Hex.Shell.info(" #{key}: #{value}") end end - defp meta_value(list) when is_list(list), + defp format_metadata_value(list) when is_list(list), do: Enum.join(list, ", ") - defp meta_value(map) when is_map(map), - do: "\n " <> Enum.map_join(map, "\n ", fn {k, v} -> "#{k}: #{v}" end) - defp meta_value(value), + defp format_metadata_value(map) when is_map(map), + do: "\n " <> Enum.map_join(map, "\n ", fn {key, val} -> "#{key}: #{val}" end) + defp format_metadata_value(value), do: value - defp missing_files(nil), do: [] - defp missing_files(files) do - Enum.filter(files, &(Path.wildcard(&1) == [])) - end - - defp error_missing!(meta) do - meta - |> missing(@error_fields ++ @warn_fields) - |> check_missing_fields() - end - - defp check_missing_fields([]), do: [] - defp check_missing_fields(fields) do - fields = Enum.join(fields, ", ") - ["Missing metadata fields: #{fields}"] + defp check_missing_fields(metadata) do + fields = @error_fields ++ @warn_fields + taken_fields = Map.take(metadata, fields) |> Map.keys + case fields -- taken_fields do + [] -> + [] + missing -> + ["Missing metadata fields: #{Enum.join(missing, ", ")}"] + end end - defp error_long_description(meta) do - description = meta[:description] || "" + defp check_description_length(metadata) do + descr = metadata[:description] || "" - if String.length(description) > @max_description_length do + if String.length(descr) > @max_description_length do ["Package description is very long (exceeds #{@max_description_length} characters)"] else [] end end - defp error_missing_files(package_files) do - package_files - |> missing_files() - |> check_missing_files() - end - - defp check_missing_files([]), do: [] - defp check_missing_files(missing) do - missing = Enum.join(missing, ", ") - ["Missing files: #{missing}"] - end - - defp missing(meta, fields) do - taken_fields = Map.take(meta, fields) |> Map.keys - fields -- taken_fields + defp check_missing_files(package_files) do + case Enum.filter(package_files, &(Path.wildcard(&1) == [])) do + [] -> + [] + missing -> + ["Missing files: #{Enum.join(missing, ", ")}"] + end end @build_tools [ @@ -234,14 +218,13 @@ defmodule Mix.Hex.Build do base_files = paths |> Enum.filter(&(Path.dirname(&1) == ".")) - |> Enum.into(HashSet.new) + |> Enum.into(Hex.Set.new) - Enum.flat_map(@build_tools, fn {file, tool} -> - if file in base_files, - do: [tool], - else: [] - end) - |> default_build_tool + for {file, tool} <- @build_tools, file in base_files do + tool + end + |> default_build_tool() + |> Enum.uniq end defp default_build_tool([]), do: ["mix"] diff --git a/lib/mix/hex/utils.ex b/lib/mix/hex/utils.ex index c3d2c2890..11785f9e9 100644 --- a/lib/mix/hex/utils.ex +++ b/lib/mix/hex/utils.ex @@ -1,9 +1,9 @@ defmodule Mix.Hex.Utils do @apikey_tag "HEXAPIKEY" - def table(header, values) do + def print_table(header, values) do header = Enum.map(header, &[:underline, &1]) - widths = widths([header|values]) + widths = widths([header | values]) print_row(header, widths) Enum.each(values, &print_row(&1, widths)) @@ -17,8 +17,9 @@ defmodule Mix.Hex.Utils do do: 0 defp print_row(strings, widths) do - Enum.map(Enum.zip(strings, widths), fn {string, width} -> - pad_size = width-ansi_length(string)+2 + Enum.zip(strings, widths) + |> Enum.map(fn {string, width} -> + pad_size = width - ansi_length(string) + 2 pad = :lists.duplicate(pad_size, ?\s) [string, :reset, pad] end) @@ -26,14 +27,14 @@ defmodule Mix.Hex.Utils do |> Hex.Shell.info end - defp widths([head|tail]) do + defp widths([head | tail]) do widths = Enum.map(head, &ansi_length/1) Enum.reduce(tail, widths, fn list, acc -> Enum.zip(list, acc) |> Enum.map(fn {string, width} -> max(width, ansi_length(string)) end) end) - end + end def generate_key(username, password) do Hex.Shell.info("Generating API key...") @@ -69,8 +70,8 @@ defmodule Mix.Hex.Utils do end def encrypt_key(config, key, challenge \\ "Passphrase") do - password = password_get("#{challenge}:") |> String.strip - confirm = password_get("#{challenge} (confirm):") |> String.strip + password = password_get("#{challenge}:") |> Hex.string_trim + confirm = password_get("#{challenge} (confirm):") |> Hex.string_trim if password != confirm do Mix.raise "Entered passphrases do not match" end @@ -84,7 +85,7 @@ defmodule Mix.Hex.Utils do end def decrypt_key(encrypted_key, challenge \\ "Passphrase") do - password = password_get("#{challenge}:") |> String.strip + password = password_get("#{challenge}:") |> Hex.string_trim case Hex.Crypto.decrypt(encrypted_key, password, @apikey_tag) do {:ok, key} -> key diff --git a/lib/mix/tasks/hex.ex b/lib/mix/tasks/hex.ex index 36eb0fbbf..ac971e3a0 100644 --- a/lib/mix/tasks/hex.ex +++ b/lib/mix/tasks/hex.ex @@ -25,6 +25,8 @@ defmodule Mix.Tasks.Hex do against the repository's public key * `HTTP_PROXY` / `HTTPS_PROXY` - Sets the URL to a HTTP(S) proxy, the environment variables can also be in lower case + * `HEX_HTTP_CONCURRENCY` - Limits the number of concurrent HTTP requests in + flight, (Default: 8) """ def run(args) do diff --git a/lib/mix/tasks/hex/build.ex b/lib/mix/tasks/hex/build.ex index 615db6519..dac4aab37 100644 --- a/lib/mix/tasks/hex/build.ex +++ b/lib/mix/tasks/hex/build.ex @@ -66,7 +66,8 @@ defmodule Mix.Tasks.Hex.Build do """ def run(_args) do - build = Build.prepare_package! + Hex.start + build = Build.prepare_package meta = build.meta package = build.package diff --git a/lib/mix/tasks/hex/config.ex b/lib/mix/tasks/hex/config.ex index eb5412911..7b0a9a82b 100644 --- a/lib/mix/tasks/hex/config.ex +++ b/lib/mix/tasks/hex/config.ex @@ -27,6 +27,8 @@ defmodule Mix.Tasks.Hex.Config do signature against the repository's public key * `http_proxy` - HTTP proxy server * `https_proxy` - HTTPS proxy server + * `http_concurrency` - Limits the number of concurrent HTTP requests in + flight, can be overridden by setting `HEX_HTTP_CONCURRENCY` ## Command line options @@ -36,10 +38,8 @@ defmodule Mix.Tasks.Hex.Config do @switches [delete: :boolean] def run(args) do - {opts, args, _} = OptionParser.parse(args, switches: @switches) - Hex.start - Hex.Utils.ensure_registry(fetch: false) + {opts, args, _} = OptionParser.parse(args, switches: @switches) case args do [] -> @@ -49,7 +49,10 @@ defmodule Mix.Tasks.Hex.Config do [key, value] -> set(key, value) _ -> - Mix.raise "Invalid arguments, expected: mix hex.config KEY [VALUE]" + Mix.raise """ + Invalid arguments, expected: + mix hex.config KEY [VALUE] + """ end end diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index f5684db66..f4e3935b8 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -4,44 +4,54 @@ defmodule Mix.Tasks.Hex.Docs do @shortdoc "Fetch or open documentation of a package" @moduledoc """ - Fetch or open documentation of a package + Fetch or open documentation of a package. - mix hex.docs fetch package + mix hex.docs fetch PACKAGE [VERSION] It will retrieve and decompress the specified version of the documentation for a package. If you do not specify the `version` argument, this task will retrieve the latest documentation available in the mirror. - mix hex.docs open package + mix hex.docs open PACKAGE [VERSION] + + ## Command line options + + * `--offline` - Open a local version available in your filesystem + * `--module Some.Module` - Open a specified module documentation page inside desired package It will open the specified version of the documentation for a package in a Web browser. If you do not specify the `version` argument, this task will - open the latest documentation available in your filesystem. + open the latest documentation. """ + @switches [offline: :boolean, module: :string] + def run(args) do Hex.start - {opts, args, _} = OptionParser.parse(args) + {opts, args, _} = OptionParser.parse(args, switches: @switches) opts = normalize_options(opts) case args do [] -> - deprecation_msg = """ - [deprecation] Calling mix hex.docs without a command is deprecated, please use: - mix hex.publish docs + Mix.raise """ + [deprecation] The "mix hex.docs" command has changed. To use the old + behaviour (publishing docs), use: + + mix hex.publish docs + + The new "mix hex.docs" command has to be invoked with at least one + argument. Call "mix help hex.docs" for more information. """ - Mix.raise deprecation_msg ["fetch" | remaining] -> fetch_docs(remaining, opts) ["open" | remaining] -> open_docs(remaining, opts) _ -> - message = """ - invalid arguments, expected one of: - mix hex.docs fetch PACKAGE [VERSION] - mix hex.docs open PACKAGE [VERSION] + Mix.raise """ + Invalid arguments, expected one of: + mix hex.docs fetch PACKAGE [VERSION] + mix hex.docs open PACKAGE [VERSION] """ - Mix.raise message end end @@ -96,50 +106,90 @@ defmodule Mix.Tasks.Hex.Docs do Mix.raise "You must specify at least the name of a package" end - defp open_docs([name], opts) do - latest_version = find_latest_version("#{opts[:home]}/#{name}") + defp open_docs(package, opts) do + if opts[:offline] do + open_docs_offline(package, opts) + else + package + |> get_docs_url(opts) + |> browser_open + end + end + + defp open_docs_offline([name], opts) do + {missing?, latest_version} = find_package_version(name, opts) + if missing? do + fetch_docs([name], opts) + end open_docs([name, latest_version], opts) end - defp open_docs([name, version], opts) do + defp open_docs_offline([name, version], opts) do index_path = Path.join([opts[:home], name, version, 'index.html']) open_file(index_path) - end + end - defp open_file(path) do - unless File.exists?(path) do - Mix.raise "Documentation file not found: #{path}" + defp find_package_version(name, opts) do + if File.exists?("#{opts[:home]}/#{name}") do + {false, find_latest_version("#{opts[:home]}/#{name}")} + else + {true, find_package_latest_version(name)} end + end + defp get_docs_url([name], opts) do + if module = opts[:module] do + Hex.Utils.hexdocs_module_url(name, module) + else + Hex.Utils.hexdocs_url(name) + end + end + + defp get_docs_url([name, version], opts) do + if module = opts[:module] do + Hex.Utils.hexdocs_module_url(name, version, module) + else + Hex.Utils.hexdocs_url(name, version) + end + end + + defp browser_open(path) do start_browser_command = case :os.type do {:win32, _} -> "start" - {:unix, _} -> + {:unix, :darwin} -> "open" + {:unix, _} -> + "xdg-open" end if System.find_executable(start_browser_command) do - System.cmd start_browser_command, [path] + System.cmd(start_browser_command, [path]) else Mix.raise "Command not found: #{start_browser_command}" end end + defp open_file(path) do + unless File.exists?(path) do + Mix.raise "Documentation file not found: #{path}" + end + + browser_open(path) + end + defp find_latest_version(path) do path - |> File.ls!() + |> File.ls! |> Enum.sort(&(Hex.Version.compare(&1, &2) == :gt)) - |> List.first() + |> List.first end defp retrieve_compressed_docs(url, filename, opts) do target = Path.join(opts[:cache], filename) - - unless File.exists?(opts[:cache]) do - File.mkdir_p! opts[:cache] - end + File.mkdir_p!(opts[:cache]) unless File.exists?(target) do request_docs_from_mirror(url, target) @@ -147,12 +197,8 @@ defmodule Mix.Tasks.Hex.Docs do end defp request_docs_from_mirror(url, target) do - case Hex.Repo.request(url, nil) do - {:ok, body} when is_binary(body) -> - File.write!(target, body) - other -> - other - end + {:ok, body, _} = Hex.Repo.request(url, nil) + File.write!(target, body) end defp extract_doc_contents(filename, target_dir, opts) do @@ -162,8 +208,12 @@ defmodule Mix.Tasks.Hex.Docs do end defp normalize_options(opts) do - docs_root_path = :home |> Hex.State.fetch!() |> Path.join("docs") - cache_dir = Path.join(docs_root_path, ".cache") - Keyword.put(opts, :home, docs_root_path) |> Keyword.put(:cache, cache_dir) + home = Hex.State.fetch!(:home) + docs_root = Path.join(home, "docs") + cache_dir = Path.join(docs_root, ".cache") + + opts + |> Keyword.put(:home, docs_root) + |> Keyword.put(:cache, cache_dir) end end diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index e94382e93..42f20db4f 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -22,46 +22,33 @@ defmodule Mix.Tasks.Hex.Info do Hex.start case args do - [] -> general() - [package] -> package(package) - [package, version] -> release(package, version) + [] -> + general() + [package] -> + package(package) + [package, version] -> + release(package, version) _ -> - Mix.raise "Invalid arguments, expected: mix hex.info [PACKAGE [VERSION]]" + Mix.raise """ + Invalid arguments, expected: + mix hex.info [PACKAGE [VERSION]] + """ end end - defp general() do + defp general do Hex.Shell.info "Hex: #{Hex.version}" Hex.Shell.info "Elixir: #{System.version}" - Hex.Shell.info "OTP: #{Hex.Utils.otp_version}\n" - Hex.Shell.info "Built with: Elixir #{Hex.elixir_version} and OTP #{Hex.otp_version}\n" - - # Make sure to fetch registry after showing Hex version. Issues with the - # registry should not prevent printing the version. - {fetch_time, _} = :timer.tc fn -> Hex.Utils.ensure_registry!(cache: false, open: false) end - {load_time, _} = :timer.tc fn -> Hex.Registry.open(Hex.Registry.ETS) end - path = Hex.Registry.ETS.path - stat = File.stat!(path, time: :local) - stat_gz = File.stat!(path <> ".gz", time: :local) - file_size = stat.size |> div(1024) - file_gz_size = stat_gz.size |> div(1024) - mem_size = Hex.Registry.ETS.memory |> div(1024) - {packages, releases} = Hex.Registry.stat() - - Hex.Shell.info "Registry file available (last updated: #{format_date(stat.mtime)})" - Hex.Shell.info "File size: #{file_size}kB (compressed #{file_gz_size}kb)" - Hex.Shell.info "Memory size: #{mem_size}kB" - Hex.Shell.info "Fetch time: #{div fetch_time, 1000}ms" - Hex.Shell.info "Load time: #{div load_time, 1000}ms" - Hex.Shell.info "Packages #: #{packages}" - Hex.Shell.info "Versions #: #{releases}" - - Hex.Registry.info_installs + Hex.Shell.info "OTP: #{Hex.Utils.otp_version}" + Hex.Shell.info "" + Hex.Shell.info "Built with: Elixir #{Hex.elixir_version} and OTP #{Hex.otp_version}" + + Hex.Registry.open!(Hex.Registry.Server) + Hex.Registry.Server.check_update + Hex.Registry.close end defp package(package) do - Hex.Utils.ensure_registry(cache: false) - case Hex.API.Package.get(package) do {code, body, _} when code in 200..299 -> print_package(body) @@ -74,8 +61,6 @@ defmodule Mix.Tasks.Hex.Info do end defp release(package, version) do - Hex.Utils.ensure_registry(cache: false) - case Hex.API.Release.get(package, version) do {code, body, _} when code in 200..299 -> print_release(package, body) @@ -98,9 +83,9 @@ defmodule Mix.Tasks.Hex.Info do end defp format_releases(releases) do - {releases, rest} = Enum.split(releases, 10) + {releases, rest} = Enum.split(releases, 8) Enum.map_join(releases, ", ", &(&1["version"])) <> - if(rest != [], do: ", ..." , else: "") + if(rest != [], do: ", ..." , else: "") end defp print_meta(meta) do @@ -121,16 +106,19 @@ defmodule Mix.Tasks.Hex.Info do if requirements = release["requirements"] do Hex.Shell.info "Dependencies:" Enum.each(requirements, fn {name, req} -> + app = req["app"] + app = if app && app != name, do: " (app: #{app})" optional = if req["optional"], do: " (optional)" - app = if (app = req["app"]) && app != name, do: " (app: #{app})" Hex.Shell.info " #{name} #{req["requirement"]}#{app}#{optional}" end) end end defp print_config(name, release) do - app_name = release["meta"]["app"] || name + app_name = String.to_atom(release["meta"]["app"] || name) + name = String.to_atom(name) {:ok, version} = Hex.Version.parse(release["version"]) + snippet = format_version(version) |> format_config_snippet(name, app_name) @@ -138,9 +126,9 @@ defmodule Mix.Tasks.Hex.Info do end defp format_config_snippet(version, name, name), - do: "{:#{name}, \"#{version}\"}" + do: "{#{inspect name}, #{inspect version}}" defp format_config_snippet(version, name, app_name), - do: "{:#{app_name}, \"#{version}\", hex: :#{name}}" + do: "{#{inspect app_name}, #{inspect version}, hex: #{inspect name}}" defp format_version(%Version{major: 0, minor: minor, patch: patch, pre: []}), do: "~> 0.#{minor}.#{patch}" @@ -159,30 +147,21 @@ defmodule Mix.Tasks.Hex.Info do end defp print_list(meta, name) do - if (list = meta[name]) && list != [] do + list = Map.get(meta, name, []) + if list != [] do Hex.Shell.info(String.capitalize(name) <> ": " <> Enum.join(list, ", ")) end end defp print_dict(meta, name) do title = String.capitalize(name) + dict = Map.get(meta, name, []) - if (dict = meta[name]) && dict != [] do + if dict != [] do Hex.Shell.info title <> ":" Enum.each(dict, fn {key, val} -> Hex.Shell.info " #{key}: #{val}" end) end end - - defp format_date({{year, month, day}, {hour, min, sec}}) do - "#{pad0(year, 4)}-#{pad0(month, 2)}-#{pad0(day, 2)} " <> - "#{pad0(hour, 2)}:#{pad0(min, 2)}:#{pad0(sec, 2)}" - end - - defp pad0(int, padding) do - str = to_string(int) - padding = max(padding - byte_size(str), 0) - String.duplicate("0", padding) <> str - end end diff --git a/lib/mix/tasks/hex/install.ex b/lib/mix/tasks/hex/install.ex new file mode 100644 index 000000000..17c9f9bed --- /dev/null +++ b/lib/mix/tasks/hex/install.ex @@ -0,0 +1,103 @@ +defmodule Mix.Tasks.Hex.Install do + use Mix.Task + + @hex_list_path "/installs/hex-1.x.csv" + @hex_archive_path "/installs/[ELIXIR_VERSION]/hex-[HEX_VERSION].ez" + @public_keys_html "https://repo.hex.pm/installs/public_keys.html" + + @shortdoc false + + @moduledoc """ + Manually install specific Hex version. + + mix hex.install VERSION + """ + + def run(args) do + Hex.start + {_, args, _} = OptionParser.parse(args) + + case args do + [version] -> + install(version) + _ -> + Mix.raise """ + Invalid arguments, expected: + mix hex.install VERSION + """ + end + end + + defp install(hex_version) do + hex_mirror = Hex.State.fetch!(:mirror) + csv_url = hex_mirror <> @hex_list_path + + case find_matching_versions_from_signed_csv!("Hex", csv_url, hex_version) do + {elixir_version, sha512} -> + archive_url = + (hex_mirror <> @hex_archive_path) + |> String.replace("[ELIXIR_VERSION]", elixir_version) + |> String.replace("[HEX_VERSION]", hex_version) + + Mix.Tasks.Archive.Install.run [archive_url, "--sha512", sha512, "--force"] + + nil -> + Mix.raise "Failed to find installation for Hex #{hex_version} and Elixir #{System.version}" + end + end + + defp find_matching_versions_from_signed_csv!(name, path, hex_version) do + csv = read_path!(name, path) + + signature = + read_path!(name, path <> ".signed") + |> String.replace("\n", "") + |> Base.decode64! + + if Mix.PublicKey.verify(csv, :sha512, signature) do + csv + |> parse_csv + |> find_eligible_version(hex_version) + else + Mix.raise "Could not install #{name} because Hex could not verify authenticity " <> + "of metadata file at #{path}. This may happen because a proxy or some " <> + "entity is interfering with the download or because you don't have a " <> + "public key to verify the download.\n\nYou may try again later or check " <> + "if a new public key has been released in our public keys page: #{@public_keys_html}" + end + end + + defp read_path!(name, path) do + case Mix.Utils.read_path(path) do + {:ok, contents} -> + contents + {:remote, message} -> + Mix.raise """ + #{message} + + Could not install #{name} because Hex could not download metadata at #{path}. + """ + end + end + + defp parse_csv(body) do + body + |> :binary.split("\n", [:global, :trim]) + |> Enum.map(&:binary.split(&1, ",", [:global, :trim])) + end + + defp find_eligible_version(entries, hex_version) do + elixir_version = Hex.Version.parse!(System.version) + + entries + |> Enum.reverse + |> Enum.find_value(&find_version(&1, elixir_version, hex_version)) + end + + defp find_version([hex_version, digest | versions], elixir_version, hex_version) do + if version = Enum.find(versions, &Version.compare(&1, elixir_version) != :gt) do + {version, digest} + end + end + defp find_version(_versions, _elixir_version, _hex_version), do: nil +end diff --git a/lib/mix/tasks/hex/key.ex b/lib/mix/tasks/hex/key.ex index 0a045c075..c336d3110 100644 --- a/lib/mix/tasks/hex/key.ex +++ b/lib/mix/tasks/hex/key.ex @@ -1,91 +1,15 @@ defmodule Mix.Tasks.Hex.Key do use Mix.Task - alias Mix.Hex.Utils - @shortdoc "Manages Hex API key" - - @moduledoc """ - Removes or lists API keys associated with your account. - - ### Remove key - - Removes given API key from account. - - The key can no longer be used to authenticate API requests. - - mix hex.key remove key_name - - To remove all API keys from your account, specify with `--all` - - mix hex.key remove --all - - ### List keys - - Lists all API keys associated with your account. - - mix hex.key list - """ - - @switches [all: :boolean] - - def run(args) do - {opts, args, _} = OptionParser.parse(args, switches: @switches) + @moduledoc false + def run(_) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - - auth = Utils.auth_info(Hex.Config.read) - - all_flag = !!opts[:all] - - case args do - ["remove", key] -> - remove_key(key, auth) - ["remove"] when all_flag === true -> - remove_all_keys(auth) - ["list"] -> - list_keys(auth) - _ -> - Mix.raise "Invalid arguments, expected one of:\nmix hex.key remove KEY\nmix hex.key remove --all\nmix hex.key list" - end - end - - defp remove_key(key, auth) do - Hex.Shell.info "Removing key #{key}..." - case Hex.API.Key.delete(key, auth) do - {200, %{"name" => ^key, "authing_key" => true}, _headers} -> - Mix.Tasks.Hex.User.run(["deauth"]) - :ok - {code, _body, _headers} when code in 200..299 -> - :ok - {code, body, _headers} -> - Hex.Shell.error "Key removal failed" - Hex.Utils.print_error_result(code, body) - end - end - - defp remove_all_keys(auth) do - Hex.Shell.info "Removing all keys..." - case Hex.API.Key.delete_all(auth) do - {code, %{"name" => _, "authing_key" => true}, _headers} when code in 200..299 -> - Mix.Tasks.Hex.User.run(["deauth"]) - :ok - {code, body, _headers} -> - Hex.Shell.error "Key removal failed" - Hex.Utils.print_error_result(code, body) - end - end - defp list_keys(auth) do - case Hex.API.Key.get(auth) do - {code, body, _headers} when code in 200..299 -> - values = Enum.map(body, fn %{"name" => name, "inserted_at" => time} -> - [name, time] - end) - Utils.table(["Name", "Created at"], values) - {code, body, _headers} -> - Hex.Shell.error "Key fetching failed" - Hex.Utils.print_error_result(code, body) - end + deprecation_msg = """ + [deprecation] The mix hex.key task is deprecated, please use: + mix hex.user + """ + Mix.raise deprecation_msg end end diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index de6906633..f6f9e109d 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -26,15 +26,15 @@ defmodule Mix.Tasks.Hex.Outdated do @switches [all: :boolean, pre: :boolean] def run(args) do - {opts, args, _} = OptionParser.parse(args, switches: @switches) Hex.start - Hex.Utils.ensure_registry!() + {opts, args, _} = OptionParser.parse(args, switches: @switches) lock = Mix.Dep.Lock.read deps = Mix.Dep.loaded([]) |> Enum.filter(& &1.scm == Hex.SCM) - # Re-open registry because loading deps cleans the process dict - Hex.Utils.ensure_registry!() + Hex.Registry.open!(Hex.Registry.Server) + Hex.Mix.packages_from_lock(lock) + |> Hex.Registry.prefetch case args do [app] -> @@ -66,29 +66,28 @@ defmodule Mix.Tasks.Hex.Outdated do requirements = if dep.top_level do - [["mix.exs", dep.requirement]|requirements] + [["mix.exs", dep.requirement] | requirements] else requirements end if outdated? do ["There is newer version of the dependency available ", :bright, latest, " > ", current, :reset, "!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> Hex.Shell.info else ["Current version ", :bright, current, :reset, " of dependency is up to date!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> Hex.Shell.info end - Hex.Shell.info "" - - header = ["Parent", "Requirement"] + header = ["Source", "Requirement"] values = Enum.map(requirements, &format_single_row(&1, latest)) - Utils.table(header, values) - Hex.Shell.info "" - Hex.Shell.info "A green requirement means that it matches the latest version." + Utils.print_table(header, values) + + message = "A green requirement means that it matches the latest version." + Hex.Shell.info ["\n", message] end defp get_requirements(deps, app) do @@ -101,16 +100,14 @@ defmodule Mix.Tasks.Hex.Outdated do end) end - defp format_single_row([parent, req], latest) do + defp format_single_row([source, req], latest) do req_matches? = version_match?(latest, req) req = req || "" req_color = if req_matches?, do: :green, else: :red - [[:bright, parent], [req_color, req]] + [[:bright, source], [req_color, req]] end defp all(deps, lock, opts) do - header = ["Dependency", "Current", "Latest", "Requirement"] - values = if(opts[:all], do: deps, else: Enum.filter(deps, & &1.top_level)) |> sort @@ -120,12 +117,15 @@ defmodule Mix.Tasks.Hex.Outdated do if Enum.empty?(values) do Hex.Shell.info "No hex dependencies" else - Utils.table(header, values) - - Hex.Shell.info "" - Hex.Shell.info "A green version in latest means you have the latest " <> - "version of a given package. A green requirement means " <> - "your current requirement matches the latest version." + header = ["Dependency", "Current", "Latest", "Update possible"] + Utils.print_table(header, values) + + message = + "A green version in latest means you have the latest " <> + "version of a given package. Update possible indicates " <> + "if your current requirement matches the latest version.\n" <> + "Run `mix hex.outdated APP` to see requirements for a specific dependency." + Hex.Shell.info ["\n" | message] end end @@ -138,8 +138,14 @@ defmodule Mix.Tasks.Hex.Outdated do case Hex.Utils.lock(lock[dep.app]) do [:hex, package, lock_version, _checksum, _managers, _deps] -> latest_version = latest_version(package, lock_version, pre?) - req = dep.requirement - [[Atom.to_string(dep.app), lock_version, latest_version, req]] + + requirements = + deps + |> get_requirements(dep.app) + |> Enum.map(fn [_, req_version] -> req_version end) + requirements = [dep.requirement | requirements] + + [[Atom.to_string(dep.app), lock_version, latest_version, requirements]] _ -> [] end @@ -153,7 +159,7 @@ defmodule Mix.Tasks.Hex.Outdated do latest = package |> Atom.to_string - |> Hex.Registry.get_versions + |> Hex.Registry.versions |> highest_version(pre?) latest || default @@ -172,18 +178,23 @@ defmodule Mix.Tasks.Hex.Outdated do List.last(versions) end - defp format_all_row([package, lock, latest, req]) do + defp format_all_row([package, lock, latest, requirements]) do outdated? = Hex.Version.compare(lock, latest) == :lt latest_color = if outdated?, do: :red, else: :green - req_matches? = version_match?(latest, req) - req = req || "" - req_color = if req_matches?, do: :green, else: :red + req_matches? = Enum.all?(requirements, &(version_match?(latest, &1))) + + {update_possible_color, update_possible} = + case {outdated?, req_matches?} do + {true, true} -> {:green, "Yes"} + {true, false} -> {:red, "No"} + {false, _} -> {:green, ""} + end [[:bright, package], lock, [latest_color, latest], - [req_color, req]] + [update_possible_color, update_possible]] end defp version_match?(_version, nil), do: true diff --git a/lib/mix/tasks/hex/owner.ex b/lib/mix/tasks/hex/owner.ex index 6dbc9e0bc..474796436 100644 --- a/lib/mix/tasks/hex/owner.ex +++ b/lib/mix/tasks/hex/owner.ex @@ -39,24 +39,28 @@ defmodule Mix.Tasks.Hex.Owner do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - config = Hex.Config.read - auth = Utils.auth_info(config) case args do ["add", package, owner] -> + auth = Utils.auth_info(config) add_owner(package, owner, auth) ["remove", package, owner] -> + auth = Utils.auth_info(config) remove_owner(package, owner, auth) ["list", package] -> + auth = Utils.auth_info(config) list_owners(package, auth) ["packages"] -> - list_owned_packages(config, auth) + list_owned_packages(config) _ -> - Mix.raise "Invalid arguments, expected one of:\nmix hex.owner add PACKAGE EMAIL\n" <> - "mix hex.owner remove PACKAGE EMAIL\nmix hex.owner list PACKAGE\n" <> - "mix hex.owner packages" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.owner add PACKAGE EMAIL + mix hex.owner remove PACKAGE EMAIL + mix hex.owner list PACKAGE + mix hex.owner packages + """ end end @@ -92,16 +96,21 @@ defmodule Mix.Tasks.Hex.Owner do end end - def list_owned_packages(config, auth) do + def list_owned_packages(config) do {:ok, username} = Keyword.fetch(config, :username) - case Hex.API.User.get(username, auth) do + + case Hex.API.User.get(username) do {code, body, _headers} when code in 200..299 -> - Enum.each(body["owned_packages"], fn({name, url}) -> - Hex.Shell.info("#{name} - #{url}") + Enum.each(body["owned_packages"], fn {name, _url} -> + Hex.Shell.info("#{name} - #{url(name)}") end) {code, body, _headers} -> Hex.Shell.error("Listing owned packages failed") Hex.Utils.print_error_result(code, body) end end + + defp url(name) do + "https://hex.pm/packages/#{name}" + end end diff --git a/lib/mix/tasks/hex/public_keys.ex b/lib/mix/tasks/hex/public_keys.ex index d90fe2726..b7dcf48af 100644 --- a/lib/mix/tasks/hex/public_keys.ex +++ b/lib/mix/tasks/hex/public_keys.ex @@ -41,8 +41,6 @@ defmodule Mix.Tasks.Hex.PublicKeys do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - {opts, args, _} = OptionParser.parse(args, switches: @switches) case args do @@ -53,41 +51,44 @@ defmodule Mix.Tasks.Hex.PublicKeys do ["remove", key|_] -> remove(key, opts) _ -> - Mix.raise "Invalid arguments, expected one of:\n" <> - "mix hex.public_keys list\n" <> - "mix hex.public_keys add URL_TO_REPO LOCAL_PATH_TO_KEY\n" <> - "mix hex.public_keys remove URL_TO_REPO" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.public_keys list + mix hex.public_keys add URL_TO_REPO LOCAL_PATH_TO_KEY + mix hex.public_keys remove URL_TO_REPO + """ end end defp list(opts) do - for {id, key} <- Hex.PublicKey.public_keys do + for {id, key} <- Hex.Crypto.PublicKey.public_keys do Hex.Shell.info "* #{id}" if opts[:detailed] do Hex.Shell.info "\n#{key}" end end - Hex.Shell.info "Public keys (except in-memory ones) installed at: #{Hex.PublicKey.public_keys_path()}" + Hex.Shell.info "Public keys (except in-memory ones) installed at: " <> + Hex.Crypto.PublicKey.public_keys_path() end defp add(id, source, opts) do data = File.read!(source) file = Base.url_encode64(id) - dest = Path.join(Hex.PublicKey.public_keys_path, file) + dest = Path.join(Hex.Crypto.PublicKey.public_keys_path, file) # Validate the key is good - _ = Hex.PublicKey.decode!(id, data) + _ = Hex.Crypto.PublicKey.decode!(id, data) if opts[:force] || should_install?(id, dest) do - File.mkdir_p!(Hex.PublicKey.public_keys_path) + File.mkdir_p!(Hex.Crypto.PublicKey.public_keys_path) File.write!(dest, data) end end defp remove(id, _opts) do file = Base.url_encode64(id) - path = Path.join(Hex.PublicKey.public_keys_path, file) + path = Path.join(Hex.Crypto.PublicKey.public_keys_path, file) if File.exists?(path) do File.rm!(path) diff --git a/lib/mix/tasks/hex/publish.ex b/lib/mix/tasks/hex/publish.ex index 7f012ffd9..930333ea3 100644 --- a/lib/mix/tasks/hex/publish.ex +++ b/lib/mix/tasks/hex/publish.ex @@ -26,7 +26,7 @@ defmodule Mix.Tasks.Hex.Publish do Documentation will be generated by running the `mix docs` task. `ex_doc` provides this task by default, but any library can be used. Or an alias can be used to extend the documentation generation. The expected result of the task - is the generated documentation located in the `docs/` directory with an + is the generated documentation located in the `doc/` directory with an `index.html` file. Note that if you want to publish a new version of your package and its @@ -37,7 +37,6 @@ defmodule Mix.Tasks.Hex.Publish do ## Command line options * `--revert VERSION` - Revert given version - * `--canonical URL` - Specify the canonical URL for the documentation ## Configuration @@ -96,37 +95,40 @@ defmodule Mix.Tasks.Hex.Publish do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) {opts, args, _} = OptionParser.parse(args, switches: @switches) - auth = Utils.auth_info(Hex.Config.read) - - build = Build.prepare_package! + config = Hex.Config.read + build = Build.prepare_package revert_version = opts[:revert] revert = !!revert_version case args do ["package"] when revert -> + auth = Utils.auth_info(config) revert_package(build, revert_version, auth) ["docs"] when revert -> + auth = Utils.auth_info(config) revert_docs(build, revert_version, auth) [] when revert -> + auth = Utils.auth_info(config) revert(build, revert_version, auth) ["package"] -> + auth = Utils.auth_info(config) if proceed?(build), do: create_package(build, auth, opts) ["docs"] -> + auth = Utils.auth_info(config) docs_task(build, opts) create_docs(build, auth, opts) [] -> + auth = Utils.auth_info(config) create(build, auth, opts) _ -> - message = """ - invalid arguments, expected one of: - mix hex.publish - mix hex.publish package - mix hex.publish docs - """ - Mix.raise message + Mix.raise """ + Invalid arguments, expected one of: + mix hex.publish + mix hex.publish package + mix hex.publish docs + """ end end @@ -189,7 +191,7 @@ defmodule Mix.Tasks.Hex.Publish do end defp print_link_to_coc() do - Hex.Shell.info "Before publishing, please read Hex Code of Conduct: https://hex.pm/policies/codeofconduct" + Hex.Shell.info "Before publishing, please read the Code of Conduct: https://hex.pm/policies/codeofconduct" end defp revert(build, version, auth) do @@ -266,7 +268,7 @@ defmodule Mix.Tasks.Hex.Publish do defp relative_path(file, dir) do Path.relative_to(file, dir) - |> String.to_char_list + |> Hex.string_to_charlist end defp docs_dir do diff --git a/lib/mix/tasks/hex/registry.ex b/lib/mix/tasks/hex/registry.ex deleted file mode 100644 index 55576df4b..000000000 --- a/lib/mix/tasks/hex/registry.ex +++ /dev/null @@ -1,65 +0,0 @@ -defmodule Mix.Tasks.Hex.Registry do - use Mix.Task - - @shortdoc "Manages the local Hex registry" - - @moduledoc """ - Tasks for working with the locally cached registry file. - - ### Fetch registry - - Updates the locally cached registry file. - - mix hex.registry fetch - - ### Dump registry - - Copies the cached registry file to the given path. - - mix hex.registry dump - - ### Load registry - - Copies given regsitry file to the cache. - - mix hex.registry load - """ - - def run(args) do - Hex.start - - case args do - ["fetch"] -> - fetch() - ["dump", path] -> - dump(path) - ["load", path] -> - load(path) - _otherwise -> - message = """ - Invalid arguments, expected one of: - mix hex.registry fetch - mix hex.registry dump - mix hex.registry load - """ - Mix.raise message - end - end - - defp fetch() do - Hex.Utils.ensure_registry!(update: true) - end - - defp dump(dest) do - path_gz = Hex.Registry.ETS.path <> ".gz" - File.cp!(path_gz, dest) - end - - defp load(source) do - path = Hex.Registry.ETS.path - path_gz = path <> ".gz" - content = File.read!(source) |> :zlib.gunzip - File.cp!(source, path_gz) - File.write!(path, content) - end -end diff --git a/lib/mix/tasks/hex/retire.ex b/lib/mix/tasks/hex/retire.ex new file mode 100644 index 000000000..0ad58c869 --- /dev/null +++ b/lib/mix/tasks/hex/retire.ex @@ -0,0 +1,80 @@ +defmodule Mix.Tasks.Hex.Retire do + use Mix.Task + alias Mix.Hex.Utils + + @shortdoc "Retires a package version" + + @moduledoc """ + Retires a package version. + + mix hex.retire PACKAGE VERSION REASON + + mix hex.retire PACKAGE VERSION --unretire + + Mark a package as retired when you no longer recommend it's usage. A retired + is still resolvable and usable but it will be flagged as retired in the + repository and a message will be displayed to users when they use the package. + + ## Retirement reasons + + * **renamed** - The package has been renamed, including the new package name + in the message + * **deprecated** - The package has been deprecated, if there's a replacing + package include it in the message + * **security** - There are security issues with this package + * **invalid** - The package is invalid, for example it does not compile correctly + * **other** - Any other reason not included above, clarify the reason in + the message + + ## Command line options + + * `--message "MESSAGE"` - Optional message (up to 140 characters) clarifying + the retirement reason + """ + + @switches [message: :string, unretire: :boolean] + + def run(args) do + Hex.start + + {opts, args, _} = OptionParser.parse(args, switches: @switches) + config = Hex.Config.read + retire? = !opts[:unretire] + + case args do + [package, version, reason] when retire? -> + auth = Utils.auth_info(config) + retire(package, version, reason, opts, auth) + [package, version] when not retire? -> + auth = Utils.auth_info(config) + unretire(package, version, auth) + _ -> + Mix.raise """ + Invalid arguments, expected one of: + mix hex.retire PACKAGE VERSION REASON + mix hex.retire PACKAGE VERSION --unretire + """ + end + end + + defp retire(package, version, reason, opts, auth) do + body = %{reason: reason, message: opts[:message]} + case Hex.API.Release.retire(package, version, body, auth) do + {code, _body, _headers} when code in 200..299 -> + :ok + {code, body, _headers} -> + Hex.Shell.error "Retiring package failed" + Hex.Utils.print_error_result(code, body) + end + end + + defp unretire(package, version, auth) do + case Hex.API.Release.unretire(package, version, auth) do + {code, _body, _headers} when code in 200..299 -> + :ok + {code, body, _headers} -> + Hex.Shell.error "Unretiring package failed" + Hex.Utils.print_error_result(code, body) + end + end +end diff --git a/lib/mix/tasks/hex/search.ex b/lib/mix/tasks/hex/search.ex index 06e8386b0..5022c45b3 100644 --- a/lib/mix/tasks/hex/search.ex +++ b/lib/mix/tasks/hex/search.ex @@ -1,5 +1,6 @@ defmodule Mix.Tasks.Hex.Search do use Mix.Task + alias Mix.Hex.Utils @shortdoc "Searches for package names" @@ -14,26 +15,34 @@ defmodule Mix.Tasks.Hex.Search do case args do [package] -> - Hex.Utils.ensure_registry!() - - Hex.Registry.search(package) + Hex.API.Package.search(package) |> lookup_packages _ -> - Mix.raise "Invalid arguments, expected: mix hex.search PACKAGE" + Mix.raise """ + Invalid arguments, expected: + mix hex.search PACKAGE + """ end end - defp lookup_packages([]) do + defp lookup_packages({200, [], _headers}) do Hex.Shell.info "No packages found" end - defp lookup_packages(packages) do - pkg_max_length = Enum.max_by(packages, &byte_size/1) |> byte_size - - Enum.each(packages, fn pkg -> - vsn = Hex.Registry.get_versions(pkg) |> List.last - pkg_name = String.ljust(pkg, pkg_max_length) - Hex.Shell.info "#{pkg_name} #{vsn}" - end) + defp lookup_packages({200, packages, _headers}) do + values = + Enum.map(packages, fn package -> + [package["name"], latest(package["releases"]), url(package["name"])] + end) + + Utils.print_table(["Package", "Version", "URL"], values) + end + + defp latest([%{"version" => version} | _]) do + version + end + + defp url(name) do + "https://hex.pm/packages/#{name}" end end diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index d8fd3f900..e88a2cb8c 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -35,6 +35,26 @@ defmodule Mix.Tasks.Hex.User do mix hex.user passphrase + ### Remove key + + Removes given API key from account. + + The key can no longer be used to authenticate API requests. + + mix hex.user key --remove key_name + + ### Remove all keys + + Remove all API keys from your account. + + mix hex.user key --remove-all + + ### List keys + + Lists all API keys associated with your account. + + mix hex.user key --list + ### Test authentication Tests if authentication works with the stored API key. @@ -46,41 +66,56 @@ defmodule Mix.Tasks.Hex.User do mix hex.user reset password """ + @switches [remove_all: :boolean, remove: :string, list: :boolean] + def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - - {_, args, _} = OptionParser.parse(args, switches: []) + config = Hex.Config.read() + {opts, args, _} = OptionParser.parse(args, switches: @switches) case args do ["register"] -> register() ["whoami"] -> - whoami() + whoami(config) ["auth"] -> create_key() ["deauth"] -> - deauth() + deauth(config) ["passphrase"] -> - passphrase() + passphrase(config) ["reset", "password"] -> reset_password() ["test"] -> - test() + test(config) + ["key"] -> + process_key_task(opts, config) _ -> - Mix.raise "Invalid arguments, expected one of:\nmix hex.user register\n" <> - "mix hex.user auth\nmix hex.user whoami\nmix hex.user deauth\nmix hex.user reset password" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.user register + mix hex.user auth + mix hex.user whoami + mix hex.user deauth + mix hex.user reset password + mix hex.user key --remove-all + mix hex.user key --remove KEY_NAME + mix hex.user key --list + """ end end - defp whoami do - config = Hex.Config.read + defp process_key_task([remove_all: true], config), do: remove_all_keys(config) + defp process_key_task([remove: key], config), do: remove_key(key, config) + defp process_key_task([list: true], config), do: list_keys(config) + + defp whoami(config) do username = local_user(config) Hex.Shell.info(username) end defp reset_password do - name = Hex.Shell.prompt("Username or Email:") |> String.strip + name = Hex.Shell.prompt("Username or Email:") |> Hex.string_trim case Hex.API.User.password_reset(name) do {code, _, _} when code in 200..299 -> @@ -92,8 +127,7 @@ defmodule Mix.Tasks.Hex.User do end end - defp deauth do - config = Hex.Config.read + defp deauth(config) do username = local_user(config) config @@ -105,9 +139,7 @@ defmodule Mix.Tasks.Hex.User do "or create a new user with `mix hex.user register`" end - defp passphrase do - config = Hex.Config.read - + defp passphrase(config) do key = cond do encrypted_key = config[:encrypted_key] -> Utils.decrypt_key(encrypted_key, "Current passphrase") @@ -124,10 +156,10 @@ defmodule Mix.Tasks.Hex.User do Hex.Shell.info("By registering an account on Hex.pm you accept all our " <> "policies and terms of service found at https://hex.pm/policies\n") - username = Hex.Shell.prompt("Username:") |> String.strip - email = Hex.Shell.prompt("Email:") |> String.strip - password = Utils.password_get("Password:") |> String.strip - confirm = Utils.password_get("Password (confirm):") |> String.strip + username = Hex.Shell.prompt("Username:") |> Hex.string_trim + email = Hex.Shell.prompt("Email:") |> Hex.string_trim + password = Utils.password_get("Password:") |> Hex.string_trim + confirm = Utils.password_get("Password (confirm):") |> Hex.string_trim if password != confirm do Mix.raise "Entered passwords do not match" @@ -150,18 +182,63 @@ defmodule Mix.Tasks.Hex.User do end defp create_key do - username = Hex.Shell.prompt("Username:") |> String.strip - password = Utils.password_get("Password:") |> String.strip + username = Hex.Shell.prompt("Username:") |> Hex.string_trim + password = Utils.password_get("Password:") |> Hex.string_trim Utils.generate_key(username, password) end - defp test do - config = Hex.Config.read + defp remove_all_keys(config) do + auth = Utils.auth_info(config) + + Hex.Shell.info "Removing all keys..." + case Hex.API.Key.delete_all(auth) do + {code, %{"name" => _, "authing_key" => true}, _headers} when code in 200..299 -> + Mix.Tasks.Hex.User.run(["deauth"]) + :ok + {code, body, _headers} -> + Hex.Shell.error "Key removal failed" + Hex.Utils.print_error_result(code, body) + end + end + + defp remove_key(key, config) do + auth = Utils.auth_info(config) + + Hex.Shell.info "Removing key #{key}..." + case Hex.API.Key.delete(key, auth) do + {200, %{"name" => ^key, "authing_key" => true}, _headers} -> + Mix.Tasks.Hex.User.run(["deauth"]) + :ok + {code, _body, _headers} when code in 200..299 -> + :ok + {code, body, _headers} -> + Hex.Shell.error "Key removal failed" + Hex.Utils.print_error_result(code, body) + end + end + + defp list_keys(config) do + auth = Utils.auth_info(config) + + case Hex.API.Key.get(auth) do + {code, body, _headers} when code in 200..299 -> + values = Enum.map(body, fn %{"name" => name, "inserted_at" => time} -> + [name, time] + end) + Utils.print_table(["Name", "Created at"], values) + {code, body, _headers} -> + Hex.Shell.error "Key fetching failed" + Hex.Utils.print_error_result(code, body) + end + end + + # TODO + defp test(config) do username = local_user(config) auth = Utils.auth_info(config) - case Hex.API.User.get(username, auth) do + case Hex.API.User.test(username, auth) do {code, _, _} when code in 200..299 -> Hex.Shell.info("Successfully authed. Your key works.") {code, body, _} -> diff --git a/mix.lock b/mix-new.lock similarity index 66% rename from mix.lock rename to mix-new.lock index cece1d1d5..9603a397a 100644 --- a/mix.lock +++ b/mix-new.lock @@ -1,5 +1,6 @@ %{"bypass": {:git, "https://github.com/PSPDFKit-labs/bypass.git", "87721c7ff56e6b307cb0e04d8a44666ce502b28b", []}, "cowboy": {:git, "https://github.com/ninenines/cowboy.git", "d08c2ab39d38c181abda279d5c2cadfac33a50c1", [tag: "1.0.4"]}, "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "45f750db410a4b08c68d142ad0af839f544c5d3d", [tag: "1.0.2"]}, - "plug": {:git, "https://github.com/elixir-lang/plug.git", "9b85c4ac7da66ea5b89ac973316f0b344effceef", [tag: "v1.1.4"]}, + "mime": {:git, "https://github.com/elixir-lang/mime.git", "5ab714e38b25a59b68bda1df7b58da499b2c3aa7", [tag: "v1.0.1"]}, + "plug": {:git, "https://github.com/elixir-lang/plug.git", "1b161d55dc383df6f9e44e08f8359a862ad70b6c", [tag: "v1.2.0"]}, "ranch": {:git, "https://github.com/ninenines/ranch.git", "a5d2efcde9a34ad38ab89a26d98ea5335e88625a", [tag: "1.2.1"]}} diff --git a/mix-old.lock b/mix-old.lock new file mode 100644 index 000000000..538872b5c --- /dev/null +++ b/mix-old.lock @@ -0,0 +1,6 @@ +%{"bypass": {:git, "https://github.com/PSPDFKit-labs/bypass.git", "87721c7ff56e6b307cb0e04d8a44666ce502b28b", []}, + "cowboy": {:git, "https://github.com/ninenines/cowboy.git", "d08c2ab39d38c181abda279d5c2cadfac33a50c1", [tag: "1.0.4"]}, + "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "45f750db410a4b08c68d142ad0af839f544c5d3d", [tag: "1.0.2"]}, + "mime": {:git, "https://github.com/elixir-lang/mime.git", "5ab714e38b25a59b68bda1df7b58da499b2c3aa7", [tag: "v1.0.1"]}, + "plug": {:git, "https://github.com/elixir-lang/plug.git", "82b3e32194f044b1329e3c6361f96c1fe89bbce6", [tag: "v1.1.6"]}, + "ranch": {:git, "https://github.com/ninenines/ranch.git", "a5d2efcde9a34ad38ab89a26d98ea5335e88625a", [tag: "1.2.1"]}} diff --git a/mix.exs b/mix.exs index e608bdbf8..901b9ceb5 100644 --- a/mix.exs +++ b/mix.exs @@ -1,12 +1,18 @@ defmodule Hex.Mixfile do use Mix.Project + @version "0.15.1-dev" + + {:ok, system_version} = Version.parse(System.version) + @elixir_version {system_version.major, system_version.minor, system_version.patch} + def project do [app: :hex, - version: "0.13.1-dev", + version: @version, elixir: "~> 1.0", aliases: aliases(), - deps: deps(), + lockfile: lockfile(@elixir_version), + deps: deps(@elixir_version), elixirc_options: elixirc_options(Mix.env), elixirc_paths: elixirc_paths(Mix.env), xref: xref()] @@ -17,21 +23,34 @@ defmodule Hex.Mixfile do mod: {Hex, []}] end - defp applications(:test), do: [:ssl, :inets, :logger] - defp applications(_), do: [:ssl, :inets] + defp applications(:prod), do: [:ssl, :inets] + defp applications(_), do: [:ssl, :inets, :logger] + + # We use different versions of plug because older plug version produces + # warnings on elixir >=1.3.0 and newer plug versions do not work on elixir <1.2.3 + defp lockfile(elixir_version) when elixir_version >= {1, 2, 3}, do: "mix-new.lock" + defp lockfile(_), do: "mix-old.lock" # Can't use hex dependencies because the elixir compiler loads dependencies # and calls the dependency SCM. This would cause us to crash if the SCM was # Hex because we have to unload Hex before compiling it. + defp deps(elixir_version) when elixir_version >= {1, 2, 3} do + [{:plug, github: "elixir-lang/plug", tag: "v1.2.0", only: :test, override: true}] ++ + deps() + end + defp deps(_) do + [{:plug, github: "elixir-lang/plug", tag: "v1.1.6", only: :test, override: true}] ++ + deps() + end + defp deps do [{:bypass, github: "PSPDFKit-labs/bypass", only: :test}, - {:plug, github: "elixir-lang/plug", tag: "v1.1.4", only: :test, override: true}, + {:mime, github: "elixir-lang/mime", tag: "v1.0.1", only: :test, override: true}, {:cowboy, github: "ninenines/cowboy", tag: "1.0.4", only: :test, override: true, manager: :rebar3}, {:cowlib, github: "ninenines/cowlib", tag: "1.0.2", only: :test, override: true, manager: :rebar3}, {:ranch, github: "ninenines/ranch", tag: "1.2.1", only: :test, override: true, manager: :rebar3}] - end + end - defp elixirc_options(:prod), do: [debug_info: false] defp elixirc_options(_), do: [] defp elixirc_paths(:test), do: ["lib", "test/support"] @@ -58,7 +77,7 @@ defmodule Hex.Mixfile do ebin = archive_ebin(archive) Code.delete_path(ebin) - {:ok, files} = :erl_prim_loader.list_dir(to_char_list(ebin)) + {:ok, files} = ebin |> :unicode.characters_to_list |> :erl_prim_loader.list_dir Enum.each(files, fn file -> file = List.to_string(file) diff --git a/release.sh b/release.sh index 682d109e5..133a6008c 100755 --- a/release.sh +++ b/release.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set -e -u function join { local IFS="$1"; shift; echo "$*"; } @@ -9,24 +11,27 @@ function join { local IFS="$1"; shift; echo "$*"; } function build { rm .tool-versions || true rm -rf _build || true + rm src/safe_erl_term.erl || true - echo "erlang ${2}\nelixir ${3}" > .tool-versions + printf "erlang ${2}\nelixir ${3}" > .tool-versions MIX_ENV=prod mix compile MIX_ENV=prod mix archive.build MIX_ENV=prod mix archive.build -o hex.ez - mv hex.ez hex-${4}.ez - mv hex-${1}.ez hex-${1}-${4}.ez + mv hex.ez "hex-${4}.ez" + mv "hex-${1}.ez" "hex-${1}-${4}.ez" } # $1 = hex version # $... = elixir version function hex_csv { - rm hex-1.x.csv || true + rm hex-1.x*.csv || true + + s3down hex-1.x.csv hex-1.x.csv - for elixir in ${@:2} + for elixir in "${@:2}" do sha=$(shasum -a 512 hex-${1}-${elixir}.ez) sha=($sha) @@ -38,37 +43,43 @@ function hex_csv { # $1 = source # $2 = target -function s3cp { - aws s3 cp ${1} s3://s3.hex.pm/installs/${2} --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" +function s3up { + aws s3 cp "${1}" "s3://s3.hex.pm/installs/${2}" --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" +} + +# $1 = source +# $2 = target +function s3down { + aws s3 cp "s3://s3.hex.pm/installs/${1}" "${2}" } # $1 = hex version # $... = elixir versions function upload { - for elixir in ${@:2} + for elixir in "${@:2}" do - s3cp hex-${elixir}.ez ${elixir}/hex.ez - s3cp hex-${1}-${elixir}.ez ${elixir}/hex-${1}.ez + s3up "hex-${elixir}.ez" "${elixir}/hex.ez" + s3up "hex-${1}-${elixir}.ez" "${elixir}/hex-${1}.ez" done # special case 1.0.0 upload - s3cp hex-1.0.0.ez hex.ez + s3up hex-1.0.0.ez hex.ez - s3cp hex-1.x.csv hex-1.x.csv - s3cp hex-1.x.csv.signed hex-1.x.csv.signed + s3up hex-1.x.csv hex-1.x.csv + s3up hex-1.x.csv.signed hex-1.x.csv.signed } # UPDATE THIS FOR EVERY RELEASE hex_version=$1 -build ${hex_version} 18.3.4.2 1.3.2 1.3.0 -build ${hex_version} 18.3.4.2 1.2.6 1.2.0 +build ${hex_version} 18.3.4.4 1.3.4 1.3.0 +build ${hex_version} 18.3.4.4 1.2.6 1.2.0 build ${hex_version} 17.5.6.9 1.1.1 1.1.0 build ${hex_version} 17.5.6.9 1.0.5 1.0.0 -hex_csv ${hex_version} 1.0.0 1.1.0 1.2.0 1.3.0 -upload ${hex_version} 1.0.0 1.1.0 1.2.0 1.3.0 +hex_csv "${hex_version}" 1.0.0 1.1.0 1.2.0 1.3.0 +upload "${hex_version}" 1.0.0 1.1.0 1.2.0 1.3.0 rm -rf _build diff --git a/release_rebar.sh b/release_rebar.sh index 9a7299674..03e533048 100755 --- a/release_rebar.sh +++ b/release_rebar.sh @@ -1,23 +1,25 @@ +#!/usr/bin/env bash + set -e -u # $1 = rebar name # $2 = rebar version # $... = elixir version function rebar_csv { - for elixir in ${@:3} + for elixir in "${@:3}" do - sha=$(shasum -a 512 ${1}) + sha=$(shasum -a 512 "${1}") sha=($sha) echo "${2},${sha},${elixir}" >> ${1}-1.x.csv done - openssl dgst -sha512 -sign "${ELIXIR_PEM}" ${1}-1.x.csv | openssl base64 > ${1}-1.x.csv.signed + openssl dgst -sha512 -sign "${ELIXIR_PEM}" "${1}-1.x.csv" | openssl base64 > ${1}-1.x.csv.signed } # $1 = source # $2 = target function s3cp { - aws s3 cp ${1} s3://s3.hex.pm/installs/${2} --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" + aws s3 cp "${1}" "s3://s3.hex.pm/installs/${2}" --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" } # $1 = rebar name @@ -26,14 +28,14 @@ function s3cp { function upload { for elixir in ${@:3} do - s3cp ${1} ${elixir}/${1} - s3cp ${1} ${elixir}/${1}-${2} + s3cp "${1}" "${elixir}/${1}" + s3cp "${1}" "${elixir}/${1}-${2}" done # special case 1.0.0 upload - s3cp ${1}-1.x.csv ${1}-1.x.csv - s3cp ${1}-1.x.csv.signed ${1}-1.x.csv.signed + s3cp "${1}-1.x.csv" "${1}-1.x.csv" + s3cp "${1}-1.x.csv.signed" "${1}-1.x.csv.signed" } # UPDATE THIS FOR EVERY RELEASE @@ -42,5 +44,5 @@ function upload { rebar_name=$1 # rebar / rebar3 rebar_version=$2 -rebar_csv ${rebar_name} ${rebar_version} 1.0.0 -upload ${rebar_name} ${rebar_version} 1.0.0 +rebar_csv "${rebar_name}" "${rebar_version}" 1.0.0 +upload "${rebar_name}" "${rebar_version}" 1.0.0 diff --git a/src/hex_pb_package.erl b/src/hex_pb_package.erl new file mode 100644 index 000000000..694ac7c09 --- /dev/null +++ b/src/hex_pb_package.erl @@ -0,0 +1,1272 @@ +%% -*- coding: utf-8 -*- +%% Automatically generated, do not edit +%% Generated by gpb_compile version 3.26.4 +-module(hex_pb_package). + +-export([encode_msg/2, encode_msg/3]). +-export([decode_msg/2, decode_msg/3]). +-export([merge_msgs/3, merge_msgs/4]). +-export([verify_msg/2, verify_msg/3]). +-export([get_msg_defs/0]). +-export([get_msg_names/0]). +-export([get_enum_names/0]). +-export([find_msg_def/1, fetch_msg_def/1]). +-export([find_enum_def/1, fetch_enum_def/1]). +-export([enum_symbol_by_value/2, enum_value_by_symbol/2]). +-export([enum_symbol_by_value_RetirementReason/1, enum_value_by_symbol_RetirementReason/1]). +-export([get_service_names/0]). +-export([get_service_def/1]). +-export([get_rpc_names/1]). +-export([find_rpc_def/2, fetch_rpc_def/2]). +-export([get_package_name/0]). +-export([gpb_version_as_string/0, gpb_version_as_list/0]). + + + +-spec encode_msg(_,atom()) -> binary(). +encode_msg(Msg, MsgName) -> + encode_msg(Msg, MsgName, []). + + +-spec encode_msg(_,atom(), list()) -> binary(). +encode_msg(Msg, MsgName, Opts) -> + verify_msg(Msg, MsgName, Opts), + TrUserData = proplists:get_value(user_data, Opts), + case MsgName of + 'RetirementStatus' -> + e_msg_RetirementStatus(Msg, TrUserData); + 'Dependency' -> e_msg_Dependency(Msg, TrUserData); + 'Release' -> e_msg_Release(Msg, TrUserData); + 'Package' -> e_msg_Package(Msg, TrUserData) + end. + + + +e_msg_RetirementStatus(Msg, TrUserData) -> + e_msg_RetirementStatus(Msg, <<>>, TrUserData). + + +e_msg_RetirementStatus(#{reason := F1} = M, Bin, + TrUserData) -> + B1 = begin + TrF1 = id(F1, TrUserData), + e_enum_RetirementReason(TrF1, <>) + end, + case M of + #{message := F2} -> + begin + TrF2 = id(F2, TrUserData), + e_type_string(TrF2, <>) + end; + _ -> B1 + end. + +e_msg_Dependency(Msg, TrUserData) -> + e_msg_Dependency(Msg, <<>>, TrUserData). + + +e_msg_Dependency(#{package := F1, requirement := F2} = + M, + Bin, TrUserData) -> + B1 = begin + TrF1 = id(F1, TrUserData), + e_type_string(TrF1, <>) + end, + B2 = begin + TrF2 = id(F2, TrUserData), + e_type_string(TrF2, <>) + end, + B3 = case M of + #{optional := F3} -> + begin + TrF3 = id(F3, TrUserData), + e_type_bool(TrF3, <>) + end; + _ -> B2 + end, + B4 = case M of + #{app := F4} -> + begin + TrF4 = id(F4, TrUserData), + e_type_string(TrF4, <>) + end; + _ -> B3 + end, + case M of + #{namespace := F5} -> + begin + TrF5 = id(F5, TrUserData), + e_type_string(TrF5, <>) + end; + _ -> B4 + end. + +e_msg_Release(Msg, TrUserData) -> + e_msg_Release(Msg, <<>>, TrUserData). + + +e_msg_Release(#{version := F1, checksum := F2, + dependencies := F3} = + M, + Bin, TrUserData) -> + B1 = begin + TrF1 = id(F1, TrUserData), + e_type_string(TrF1, <>) + end, + B2 = begin + TrF2 = id(F2, TrUserData), + e_type_bytes(TrF2, <>) + end, + B3 = begin + TrF3 = id(F3, TrUserData), + if TrF3 == [] -> B2; + true -> + e_field_Release_dependencies(TrF3, B2, TrUserData) + end + end, + case M of + #{retired := F4} -> + begin + TrF4 = id(F4, TrUserData), + e_mfield_Release_retired(TrF4, <>, + TrUserData) + end; + _ -> B3 + end. + +e_msg_Package(Msg, TrUserData) -> + e_msg_Package(Msg, <<>>, TrUserData). + + +e_msg_Package(#{releases := F1}, Bin, TrUserData) -> + begin + TrF1 = id(F1, TrUserData), + if TrF1 == [] -> Bin; + true -> e_field_Package_releases(TrF1, Bin, TrUserData) + end + end. + +e_mfield_Release_dependencies(Msg, Bin, TrUserData) -> + SubBin = e_msg_Dependency(Msg, <<>>, TrUserData), + Bin2 = e_varint(byte_size(SubBin), Bin), + <>. + +e_field_Release_dependencies([Elem | Rest], Bin, + TrUserData) -> + Bin2 = <>, + Bin3 = e_mfield_Release_dependencies(id(Elem, + TrUserData), + Bin2, TrUserData), + e_field_Release_dependencies(Rest, Bin3, TrUserData); +e_field_Release_dependencies([], Bin, _TrUserData) -> + Bin. + +e_mfield_Release_retired(Msg, Bin, TrUserData) -> + SubBin = e_msg_RetirementStatus(Msg, <<>>, TrUserData), + Bin2 = e_varint(byte_size(SubBin), Bin), + <>. + +e_mfield_Package_releases(Msg, Bin, TrUserData) -> + SubBin = e_msg_Release(Msg, <<>>, TrUserData), + Bin2 = e_varint(byte_size(SubBin), Bin), + <>. + +e_field_Package_releases([Elem | Rest], Bin, + TrUserData) -> + Bin2 = <>, + Bin3 = e_mfield_Package_releases(id(Elem, TrUserData), + Bin2, TrUserData), + e_field_Package_releases(Rest, Bin3, TrUserData); +e_field_Package_releases([], Bin, _TrUserData) -> Bin. + + + +e_enum_RetirementReason('RETIRED_OTHER', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_INVALID', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_SECURITY', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_DEPRECATED', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_RENAMED', Bin) -> + <>; +e_enum_RetirementReason(V, Bin) -> e_varint(V, Bin). + +e_type_bool(true, Bin) -> <>; +e_type_bool(false, Bin) -> <>; +e_type_bool(1, Bin) -> <>; +e_type_bool(0, Bin) -> <>. + +e_type_string(S, Bin) -> + Utf8 = unicode:characters_to_binary(S), + Bin2 = e_varint(byte_size(Utf8), Bin), + <>. + +e_type_bytes(Bytes, Bin) when is_binary(Bytes) -> + Bin2 = e_varint(byte_size(Bytes), Bin), + <>; +e_type_bytes(Bytes, Bin) when is_list(Bytes) -> + BytesBin = iolist_to_binary(Bytes), + Bin2 = e_varint(byte_size(BytesBin), Bin), + <>. + +e_varint(N, Bin) when N =< 127 -> <>; +e_varint(N, Bin) -> + Bin2 = <>, + e_varint(N bsr 7, Bin2). + + + +decode_msg(Bin, MsgName) when is_binary(Bin) -> + decode_msg(Bin, MsgName, []). + +decode_msg(Bin, MsgName, Opts) when is_binary(Bin) -> + TrUserData = proplists:get_value(user_data, Opts), + case MsgName of + 'RetirementStatus' -> + d_msg_RetirementStatus(Bin, TrUserData); + 'Dependency' -> d_msg_Dependency(Bin, TrUserData); + 'Release' -> d_msg_Release(Bin, TrUserData); + 'Package' -> d_msg_Package(Bin, TrUserData) + end. + + + +d_msg_RetirementStatus(Bin, TrUserData) -> + dfp_read_field_def_RetirementStatus(Bin, 0, 0, + id('$undef', TrUserData), + id('$undef', TrUserData), TrUserData). + +dfp_read_field_def_RetirementStatus(<<8, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + d_field_RetirementStatus_reason(Rest, Z1, Z2, F1, F2, + TrUserData); +dfp_read_field_def_RetirementStatus(<<18, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + d_field_RetirementStatus_message(Rest, Z1, Z2, F1, F2, + TrUserData); +dfp_read_field_def_RetirementStatus(<<>>, 0, 0, F1, F2, + _) -> + S1 = #{reason => F1}, + if F2 == '$undef' -> S1; + true -> S1#{message => F2} + end; +dfp_read_field_def_RetirementStatus(Other, Z1, Z2, F1, + F2, TrUserData) -> + dg_read_field_def_RetirementStatus(Other, Z1, Z2, F1, + F2, TrUserData). + +dg_read_field_def_RetirementStatus(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 32 - 7 -> + dg_read_field_def_RetirementStatus(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +dg_read_field_def_RetirementStatus(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) -> + Key = X bsl N + Acc, + case Key of + 8 -> + d_field_RetirementStatus_reason(Rest, 0, 0, F1, F2, + TrUserData); + 18 -> + d_field_RetirementStatus_message(Rest, 0, 0, F1, F2, + TrUserData); + _ -> + case Key band 7 of + 0 -> + skip_varint_RetirementStatus(Rest, 0, 0, F1, F2, + TrUserData); + 1 -> + skip_64_RetirementStatus(Rest, 0, 0, F1, F2, + TrUserData); + 2 -> + skip_length_delimited_RetirementStatus(Rest, 0, 0, F1, + F2, TrUserData); + 5 -> + skip_32_RetirementStatus(Rest, 0, 0, F1, F2, TrUserData) + end + end; +dg_read_field_def_RetirementStatus(<<>>, 0, 0, F1, F2, + _) -> + S1 = #{reason => F1}, + if F2 == '$undef' -> S1; + true -> S1#{message => F2} + end. + +d_field_RetirementStatus_reason(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 57 -> + d_field_RetirementStatus_reason(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +d_field_RetirementStatus_reason(<<0:1, X:7, + Rest/binary>>, + N, Acc, _, F2, TrUserData) -> + <> = <<(X bsl N + + Acc):32/unsigned-native>>, + NewFValue = d_enum_RetirementReason(Tmp), + dfp_read_field_def_RetirementStatus(Rest, 0, 0, + NewFValue, F2, TrUserData). + + +d_field_RetirementStatus_message(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 57 -> + d_field_RetirementStatus_message(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +d_field_RetirementStatus_message(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, _, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_RetirementStatus(Rest2, 0, 0, F1, + NewFValue, TrUserData). + + +skip_varint_RetirementStatus(<<1:1, _:7, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + skip_varint_RetirementStatus(Rest, Z1, Z2, F1, F2, + TrUserData); +skip_varint_RetirementStatus(<<0:1, _:7, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + dfp_read_field_def_RetirementStatus(Rest, Z1, Z2, F1, + F2, TrUserData). + + +skip_length_delimited_RetirementStatus(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 57 -> + skip_length_delimited_RetirementStatus(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +skip_length_delimited_RetirementStatus(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_RetirementStatus(Rest2, 0, 0, F1, F2, + TrUserData). + + +skip_32_RetirementStatus(<<_:32, Rest/binary>>, Z1, Z2, + F1, F2, TrUserData) -> + dfp_read_field_def_RetirementStatus(Rest, Z1, Z2, F1, + F2, TrUserData). + + +skip_64_RetirementStatus(<<_:64, Rest/binary>>, Z1, Z2, + F1, F2, TrUserData) -> + dfp_read_field_def_RetirementStatus(Rest, Z1, Z2, F1, + F2, TrUserData). + + +d_msg_Dependency(Bin, TrUserData) -> + dfp_read_field_def_Dependency(Bin, 0, 0, + id('$undef', TrUserData), + id('$undef', TrUserData), + id('$undef', TrUserData), + id('$undef', TrUserData), + id('$undef', TrUserData), TrUserData). + +dfp_read_field_def_Dependency(<<10, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_package(Rest, Z1, Z2, F1, F2, F3, F4, + F5, TrUserData); +dfp_read_field_def_Dependency(<<18, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_requirement(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData); +dfp_read_field_def_Dependency(<<24, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_optional(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData); +dfp_read_field_def_Dependency(<<34, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_app(Rest, Z1, Z2, F1, F2, F3, F4, F5, + TrUserData); +dfp_read_field_def_Dependency(<<42, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_namespace(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData); +dfp_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, + F4, F5, _) -> + S1 = #{package => F1, requirement => F2}, + S2 = if F3 == '$undef' -> S1; + true -> S1#{optional => F3} + end, + S3 = if F4 == '$undef' -> S2; + true -> S2#{app => F4} + end, + if F5 == '$undef' -> S3; + true -> S3#{namespace => F5} + end; +dfp_read_field_def_Dependency(Other, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData) -> + dg_read_field_def_Dependency(Other, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData). + +dg_read_field_def_Dependency(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 32 - 7 -> + dg_read_field_def_Dependency(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, F5, TrUserData); +dg_read_field_def_Dependency(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) -> + Key = X bsl N + Acc, + case Key of + 10 -> + d_field_Dependency_package(Rest, 0, 0, F1, F2, F3, F4, + F5, TrUserData); + 18 -> + d_field_Dependency_requirement(Rest, 0, 0, F1, F2, F3, + F4, F5, TrUserData); + 24 -> + d_field_Dependency_optional(Rest, 0, 0, F1, F2, F3, F4, + F5, TrUserData); + 34 -> + d_field_Dependency_app(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData); + 42 -> + d_field_Dependency_namespace(Rest, 0, 0, F1, F2, F3, F4, + F5, TrUserData); + _ -> + case Key band 7 of + 0 -> + skip_varint_Dependency(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData); + 1 -> + skip_64_Dependency(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData); + 2 -> + skip_length_delimited_Dependency(Rest, 0, 0, F1, F2, F3, + F4, F5, TrUserData); + 5 -> + skip_32_Dependency(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData) + end + end; +dg_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, F4, + F5, _) -> + S1 = #{package => F1, requirement => F2}, + S2 = if F3 == '$undef' -> S1; + true -> S1#{optional => F3} + end, + S3 = if F4 == '$undef' -> S2; + true -> S2#{app => F4} + end, + if F5 == '$undef' -> S3; + true -> S3#{namespace => F5} + end. + +d_field_Dependency_package(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + d_field_Dependency_package(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, F5, TrUserData); +d_field_Dependency_package(<<0:1, X:7, Rest/binary>>, N, + Acc, _, F2, F3, F4, F5, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, NewFValue, + F2, F3, F4, F5, TrUserData). + + +d_field_Dependency_requirement(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + d_field_Dependency_requirement(Rest, N + 7, + X bsl N + Acc, F1, F2, F3, F4, F5, + TrUserData); +d_field_Dependency_requirement(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, _, F3, F4, F5, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, + NewFValue, F3, F4, F5, TrUserData). + + +d_field_Dependency_optional(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + d_field_Dependency_optional(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, F5, TrUserData); +d_field_Dependency_optional(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, _, F4, F5, TrUserData) -> + NewFValue = X bsl N + Acc =/= 0, + dfp_read_field_def_Dependency(Rest, 0, 0, F1, F2, + NewFValue, F4, F5, TrUserData). + + +d_field_Dependency_app(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + d_field_Dependency_app(Rest, N + 7, X bsl N + Acc, F1, + F2, F3, F4, F5, TrUserData); +d_field_Dependency_app(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, _, F5, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, + NewFValue, F5, TrUserData). + + +d_field_Dependency_namespace(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + d_field_Dependency_namespace(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, F5, TrUserData); +d_field_Dependency_namespace(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, _, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, + F4, NewFValue, TrUserData). + + +skip_varint_Dependency(<<1:1, _:7, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + skip_varint_Dependency(Rest, Z1, Z2, F1, F2, F3, F4, F5, + TrUserData); +skip_varint_Dependency(<<0:1, _:7, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData). + + +skip_length_delimited_Dependency(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + skip_length_delimited_Dependency(Rest, N + 7, + X bsl N + Acc, F1, F2, F3, F4, F5, + TrUserData); +skip_length_delimited_Dependency(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, + F4, F5, TrUserData). + + +skip_32_Dependency(<<_:32, Rest/binary>>, Z1, Z2, F1, + F2, F3, F4, F5, TrUserData) -> + dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData). + + +skip_64_Dependency(<<_:64, Rest/binary>>, Z1, Z2, F1, + F2, F3, F4, F5, TrUserData) -> + dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData). + + +d_msg_Release(Bin, TrUserData) -> + dfp_read_field_def_Release(Bin, 0, 0, + id('$undef', TrUserData), + id('$undef', TrUserData), id([], TrUserData), + id('$undef', TrUserData), TrUserData). + +dfp_read_field_def_Release(<<10, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + d_field_Release_version(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); +dfp_read_field_def_Release(<<18, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + d_field_Release_checksum(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); +dfp_read_field_def_Release(<<26, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + d_field_Release_dependencies(Rest, Z1, Z2, F1, F2, F3, + F4, TrUserData); +dfp_read_field_def_Release(<<34, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + d_field_Release_retired(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); +dfp_read_field_def_Release(<<>>, 0, 0, F1, F2, F3, F4, + TrUserData) -> + S1 = #{version => F1, checksum => F2, + dependencies => lists_reverse(F3, TrUserData)}, + if F4 == '$undef' -> S1; + true -> S1#{retired => F4} + end; +dfp_read_field_def_Release(Other, Z1, Z2, F1, F2, F3, + F4, TrUserData) -> + dg_read_field_def_Release(Other, Z1, Z2, F1, F2, F3, F4, + TrUserData). + +dg_read_field_def_Release(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) + when N < 32 - 7 -> + dg_read_field_def_Release(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, TrUserData); +dg_read_field_def_Release(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) -> + Key = X bsl N + Acc, + case Key of + 10 -> + d_field_Release_version(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 18 -> + d_field_Release_checksum(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 26 -> + d_field_Release_dependencies(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 34 -> + d_field_Release_retired(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + _ -> + case Key band 7 of + 0 -> + skip_varint_Release(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 1 -> + skip_64_Release(Rest, 0, 0, F1, F2, F3, F4, TrUserData); + 2 -> + skip_length_delimited_Release(Rest, 0, 0, F1, F2, F3, + F4, TrUserData); + 5 -> + skip_32_Release(Rest, 0, 0, F1, F2, F3, F4, TrUserData) + end + end; +dg_read_field_def_Release(<<>>, 0, 0, F1, F2, F3, F4, + TrUserData) -> + S1 = #{version => F1, checksum => F2, + dependencies => lists_reverse(F3, TrUserData)}, + if F4 == '$undef' -> S1; + true -> S1#{retired => F4} + end. + +d_field_Release_version(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) + when N < 57 -> + d_field_Release_version(Rest, N + 7, X bsl N + Acc, F1, + F2, F3, F4, TrUserData); +d_field_Release_version(<<0:1, X:7, Rest/binary>>, N, + Acc, _, F2, F3, F4, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Release(Rest2, 0, 0, NewFValue, F2, + F3, F4, TrUserData). + + +d_field_Release_checksum(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) + when N < 57 -> + d_field_Release_checksum(Rest, N + 7, X bsl N + Acc, F1, + F2, F3, F4, TrUserData); +d_field_Release_checksum(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, _, F3, F4, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Release(Rest2, 0, 0, F1, NewFValue, + F3, F4, TrUserData). + + +d_field_Release_dependencies(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, TrUserData) + when N < 57 -> + d_field_Release_dependencies(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, TrUserData); +d_field_Release_dependencies(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = id(d_msg_Dependency(Bs, TrUserData), + TrUserData), + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, + cons(NewFValue, F3, TrUserData), F4, TrUserData). + + +d_field_Release_retired(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) + when N < 57 -> + d_field_Release_retired(Rest, N + 7, X bsl N + Acc, F1, + F2, F3, F4, TrUserData); +d_field_Release_retired(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = id(d_msg_RetirementStatus(Bs, TrUserData), + TrUserData), + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, F3, + if F4 =:= '$undef' -> NewFValue; + true -> + merge_msg_RetirementStatus(F4, NewFValue, + TrUserData) + end, + TrUserData). + + +skip_varint_Release(<<1:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + skip_varint_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); +skip_varint_Release(<<0:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData). + + +skip_length_delimited_Release(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, TrUserData) + when N < 57 -> + skip_length_delimited_Release(Rest, N + 7, + X bsl N + Acc, F1, F2, F3, F4, TrUserData); +skip_length_delimited_Release(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, TrUserData) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, F3, F4, + TrUserData). + + +skip_32_Release(<<_:32, Rest/binary>>, Z1, Z2, F1, F2, + F3, F4, TrUserData) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData). + + +skip_64_Release(<<_:64, Rest/binary>>, Z1, Z2, F1, F2, + F3, F4, TrUserData) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData). + + +d_msg_Package(Bin, TrUserData) -> + dfp_read_field_def_Package(Bin, 0, 0, + id([], TrUserData), TrUserData). + +dfp_read_field_def_Package(<<10, Rest/binary>>, Z1, Z2, + F1, TrUserData) -> + d_field_Package_releases(Rest, Z1, Z2, F1, TrUserData); +dfp_read_field_def_Package(<<>>, 0, 0, F1, + TrUserData) -> + #{releases => lists_reverse(F1, TrUserData)}; +dfp_read_field_def_Package(Other, Z1, Z2, F1, + TrUserData) -> + dg_read_field_def_Package(Other, Z1, Z2, F1, + TrUserData). + +dg_read_field_def_Package(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, TrUserData) + when N < 32 - 7 -> + dg_read_field_def_Package(Rest, N + 7, X bsl N + Acc, + F1, TrUserData); +dg_read_field_def_Package(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, TrUserData) -> + Key = X bsl N + Acc, + case Key of + 10 -> + d_field_Package_releases(Rest, 0, 0, F1, TrUserData); + _ -> + case Key band 7 of + 0 -> skip_varint_Package(Rest, 0, 0, F1, TrUserData); + 1 -> skip_64_Package(Rest, 0, 0, F1, TrUserData); + 2 -> + skip_length_delimited_Package(Rest, 0, 0, F1, + TrUserData); + 5 -> skip_32_Package(Rest, 0, 0, F1, TrUserData) + end + end; +dg_read_field_def_Package(<<>>, 0, 0, F1, TrUserData) -> + #{releases => lists_reverse(F1, TrUserData)}. + +d_field_Package_releases(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, TrUserData) + when N < 57 -> + d_field_Package_releases(Rest, N + 7, X bsl N + Acc, F1, + TrUserData); +d_field_Package_releases(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = id(d_msg_Release(Bs, TrUserData), + TrUserData), + dfp_read_field_def_Package(Rest2, 0, 0, + cons(NewFValue, F1, TrUserData), TrUserData). + + +skip_varint_Package(<<1:1, _:7, Rest/binary>>, Z1, Z2, + F1, TrUserData) -> + skip_varint_Package(Rest, Z1, Z2, F1, TrUserData); +skip_varint_Package(<<0:1, _:7, Rest/binary>>, Z1, Z2, + F1, TrUserData) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1, + TrUserData). + + +skip_length_delimited_Package(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, TrUserData) + when N < 57 -> + skip_length_delimited_Package(Rest, N + 7, + X bsl N + Acc, F1, TrUserData); +skip_length_delimited_Package(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, TrUserData) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Package(Rest2, 0, 0, F1, TrUserData). + + +skip_32_Package(<<_:32, Rest/binary>>, Z1, Z2, F1, + TrUserData) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1, + TrUserData). + + +skip_64_Package(<<_:64, Rest/binary>>, Z1, Z2, F1, + TrUserData) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1, + TrUserData). + + + + +d_enum_RetirementReason(0) -> 'RETIRED_OTHER'; +d_enum_RetirementReason(1) -> 'RETIRED_INVALID'; +d_enum_RetirementReason(2) -> 'RETIRED_SECURITY'; +d_enum_RetirementReason(3) -> 'RETIRED_DEPRECATED'; +d_enum_RetirementReason(4) -> 'RETIRED_RENAMED'; +d_enum_RetirementReason(V) -> V. + + + +merge_msgs(Prev, New, MsgName) -> + merge_msgs(Prev, New, MsgName, []). + +merge_msgs(Prev, New, MsgName, Opts) -> + TrUserData = proplists:get_value(user_data, Opts), + case MsgName of + 'RetirementStatus' -> + merge_msg_RetirementStatus(Prev, New, TrUserData); + 'Dependency' -> + merge_msg_Dependency(Prev, New, TrUserData); + 'Release' -> merge_msg_Release(Prev, New, TrUserData); + 'Package' -> merge_msg_Package(Prev, New, TrUserData) + end. + +merge_msg_RetirementStatus(#{} = PMsg, + #{reason := NFreason} = NMsg, _) -> + S1 = #{reason => NFreason}, + case {PMsg, NMsg} of + {_, #{message := NFmessage}} -> + S1#{message => NFmessage}; + {#{message := PFmessage}, _} -> + S1#{message => PFmessage}; + _ -> S1 + end. + +merge_msg_Dependency(#{} = PMsg, + #{package := NFpackage, requirement := NFrequirement} = + NMsg, + _) -> + S1 = #{package => NFpackage, + requirement => NFrequirement}, + S2 = case {PMsg, NMsg} of + {_, #{optional := NFoptional}} -> + S1#{optional => NFoptional}; + {#{optional := PFoptional}, _} -> + S1#{optional => PFoptional}; + _ -> S1 + end, + S3 = case {PMsg, NMsg} of + {_, #{app := NFapp}} -> S2#{app => NFapp}; + {#{app := PFapp}, _} -> S2#{app => PFapp}; + _ -> S2 + end, + case {PMsg, NMsg} of + {_, #{namespace := NFnamespace}} -> + S3#{namespace => NFnamespace}; + {#{namespace := PFnamespace}, _} -> + S3#{namespace => PFnamespace}; + _ -> S3 + end. + +merge_msg_Release(#{dependencies := PFdependencies} = + PMsg, + #{version := NFversion, checksum := NFchecksum, + dependencies := NFdependencies} = + NMsg, + TrUserData) -> + S1 = #{version => NFversion, checksum => NFchecksum, + dependencies => + 'erlang_++'(PFdependencies, NFdependencies, + TrUserData)}, + case {PMsg, NMsg} of + {#{retired := PFretired}, #{retired := NFretired}} -> + S1#{retired => + merge_msg_RetirementStatus(PFretired, NFretired, + TrUserData)}; + {_, #{retired := NFretired}} -> + S1#{retired => NFretired}; + {#{retired := PFretired}, _} -> + S1#{retired => PFretired}; + {_, _} -> S1 + end. + +merge_msg_Package(#{releases := PFreleases}, + #{releases := NFreleases}, TrUserData) -> + #{releases => + 'erlang_++'(PFreleases, NFreleases, TrUserData)}. + + + +verify_msg(Msg, MsgName) -> + verify_msg(Msg, MsgName, []). + +verify_msg(Msg, MsgName, Opts) -> + TrUserData = proplists:get_value(user_data, Opts), + case MsgName of + 'RetirementStatus' -> + v_msg_RetirementStatus(Msg, ['RetirementStatus'], + TrUserData); + 'Dependency' -> + v_msg_Dependency(Msg, ['Dependency'], TrUserData); + 'Release' -> + v_msg_Release(Msg, ['Release'], TrUserData); + 'Package' -> + v_msg_Package(Msg, ['Package'], TrUserData); + _ -> mk_type_error(not_a_known_message, Msg, []) + end. + + +v_msg_RetirementStatus(#{reason := F1} = M, Path, _) -> + v_enum_RetirementReason(F1, [reason | Path]), + case M of + #{message := F2} -> v_type_string(F2, [message | Path]); + _ -> ok + end, + ok; +v_msg_RetirementStatus(M, Path, _TrUserData) + when is_map(M) -> + mk_type_error({missing_fields, [reason] -- maps:keys(M), + 'RetirementStatus'}, + M, Path); +v_msg_RetirementStatus(X, Path, _TrUserData) -> + mk_type_error({expected_msg, 'RetirementStatus'}, X, + Path). + +v_msg_Dependency(#{package := F1, requirement := F2} = + M, + Path, _) -> + v_type_string(F1, [package | Path]), + v_type_string(F2, [requirement | Path]), + case M of + #{optional := F3} -> v_type_bool(F3, [optional | Path]); + _ -> ok + end, + case M of + #{app := F4} -> v_type_string(F4, [app | Path]); + _ -> ok + end, + case M of + #{namespace := F5} -> + v_type_string(F5, [namespace | Path]); + _ -> ok + end, + ok; +v_msg_Dependency(M, Path, _TrUserData) when is_map(M) -> + mk_type_error({missing_fields, + [package, requirement] -- maps:keys(M), 'Dependency'}, + M, Path); +v_msg_Dependency(X, Path, _TrUserData) -> + mk_type_error({expected_msg, 'Dependency'}, X, Path). + +v_msg_Release(#{version := F1, checksum := F2, + dependencies := F3} = + M, + Path, TrUserData) -> + v_type_string(F1, [version | Path]), + v_type_bytes(F2, [checksum | Path]), + if is_list(F3) -> + _ = [v_msg_Dependency(Elem, [dependencies | Path], + TrUserData) + || Elem <- F3], + ok; + true -> + mk_type_error({invalid_list_of, {msg, 'Dependency'}}, + F3, Path) + end, + case M of + #{retired := F4} -> + v_msg_RetirementStatus(F4, [retired | Path], + TrUserData); + _ -> ok + end, + ok; +v_msg_Release(M, Path, _TrUserData) when is_map(M) -> + mk_type_error({missing_fields, + [version, checksum, dependencies] -- maps:keys(M), + 'Release'}, + M, Path); +v_msg_Release(X, Path, _TrUserData) -> + mk_type_error({expected_msg, 'Release'}, X, Path). + +v_msg_Package(#{releases := F1}, Path, TrUserData) -> + if is_list(F1) -> + _ = [v_msg_Release(Elem, [releases | Path], TrUserData) + || Elem <- F1], + ok; + true -> + mk_type_error({invalid_list_of, {msg, 'Release'}}, F1, + Path) + end, + ok; +v_msg_Package(M, Path, _TrUserData) when is_map(M) -> + mk_type_error({missing_fields, + [releases] -- maps:keys(M), 'Package'}, + M, Path); +v_msg_Package(X, Path, _TrUserData) -> + mk_type_error({expected_msg, 'Package'}, X, Path). + +v_enum_RetirementReason('RETIRED_OTHER', _Path) -> ok; +v_enum_RetirementReason('RETIRED_INVALID', _Path) -> ok; +v_enum_RetirementReason('RETIRED_SECURITY', _Path) -> + ok; +v_enum_RetirementReason('RETIRED_DEPRECATED', _Path) -> + ok; +v_enum_RetirementReason('RETIRED_RENAMED', _Path) -> ok; +v_enum_RetirementReason(V, Path) when is_integer(V) -> + v_type_sint32(V, Path); +v_enum_RetirementReason(X, Path) -> + mk_type_error({invalid_enum, 'RetirementReason'}, X, + Path). + +v_type_sint32(N, _Path) + when -2147483648 =< N, N =< 2147483647 -> + ok; +v_type_sint32(N, Path) when is_integer(N) -> + mk_type_error({value_out_of_range, sint32, signed, 32}, + N, Path); +v_type_sint32(X, Path) -> + mk_type_error({bad_integer, sint32, signed, 32}, X, + Path). + +v_type_bool(false, _Path) -> ok; +v_type_bool(true, _Path) -> ok; +v_type_bool(0, _Path) -> ok; +v_type_bool(1, _Path) -> ok; +v_type_bool(X, Path) -> + mk_type_error(bad_boolean_value, X, Path). + +v_type_string(S, Path) when is_list(S); is_binary(S) -> + try unicode:characters_to_binary(S) of + B when is_binary(B) -> ok; + {error, _, _} -> + mk_type_error(bad_unicode_string, S, Path) + catch + error:badarg -> + mk_type_error(bad_unicode_string, S, Path) + end; +v_type_string(X, Path) -> + mk_type_error(bad_unicode_string, X, Path). + +v_type_bytes(B, _Path) when is_binary(B) -> ok; +v_type_bytes(B, _Path) when is_list(B) -> ok; +v_type_bytes(X, Path) -> + mk_type_error(bad_binary_value, X, Path). + +-spec mk_type_error(_, _, list()) -> no_return(). +mk_type_error(Error, ValueSeen, Path) -> + Path2 = prettify_path(Path), + erlang:error({gpb_type_error, + {Error, [{value, ValueSeen}, {path, Path2}]}}). + + +prettify_path([]) -> top_level; +prettify_path(PathR) -> + list_to_atom(string:join(lists:map(fun atom_to_list/1, + lists:reverse(PathR)), + ".")). + + + +-compile({inline,id/2}). +id(X, _TrUserData) -> X. + +-compile({inline,cons/3}). +cons(Elem, Acc, _TrUserData) -> [Elem | Acc]. + +-compile({inline,lists_reverse/2}). +'lists_reverse'(L, _TrUserData) -> lists:reverse(L). +-compile({inline,'erlang_++'/3}). +'erlang_++'(A, B, _TrUserData) -> A ++ B. + + + +get_msg_defs() -> + [{{enum, 'RetirementReason'}, + [{'RETIRED_OTHER', 0}, {'RETIRED_INVALID', 1}, + {'RETIRED_SECURITY', 2}, {'RETIRED_DEPRECATED', 3}, + {'RETIRED_RENAMED', 4}]}, + {{msg, 'RetirementStatus'}, + [#{name => reason, fnum => 1, rnum => 2, + type => {enum, 'RetirementReason'}, + occurrence => required, opts => []}, + #{name => message, fnum => 2, rnum => 3, type => string, + occurrence => optional, opts => []}]}, + {{msg, 'Dependency'}, + [#{name => package, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => requirement, fnum => 2, rnum => 3, + type => string, occurrence => required, opts => []}, + #{name => optional, fnum => 3, rnum => 4, type => bool, + occurrence => optional, opts => []}, + #{name => app, fnum => 4, rnum => 5, type => string, + occurrence => optional, opts => []}, + #{name => namespace, fnum => 5, rnum => 6, + type => string, occurrence => optional, opts => []}]}, + {{msg, 'Release'}, + [#{name => version, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => checksum, fnum => 2, rnum => 3, type => bytes, + occurrence => required, opts => []}, + #{name => dependencies, fnum => 3, rnum => 4, + type => {msg, 'Dependency'}, occurrence => repeated, + opts => []}, + #{name => retired, fnum => 4, rnum => 5, + type => {msg, 'RetirementStatus'}, + occurrence => optional, opts => []}]}, + {{msg, 'Package'}, + [#{name => releases, fnum => 1, rnum => 2, + type => {msg, 'Release'}, occurrence => repeated, + opts => []}]}]. + + +get_msg_names() -> + ['RetirementStatus', 'Dependency', 'Release', + 'Package']. + + +get_enum_names() -> ['RetirementReason']. + + +fetch_msg_def(MsgName) -> + case find_msg_def(MsgName) of + Fs when is_list(Fs) -> Fs; + error -> erlang:error({no_such_msg, MsgName}) + end. + + +fetch_enum_def(EnumName) -> + case find_enum_def(EnumName) of + Es when is_list(Es) -> Es; + error -> erlang:error({no_such_enum, EnumName}) + end. + + +find_msg_def('RetirementStatus') -> + [#{name => reason, fnum => 1, rnum => 2, + type => {enum, 'RetirementReason'}, + occurrence => required, opts => []}, + #{name => message, fnum => 2, rnum => 3, type => string, + occurrence => optional, opts => []}]; +find_msg_def('Dependency') -> + [#{name => package, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => requirement, fnum => 2, rnum => 3, + type => string, occurrence => required, opts => []}, + #{name => optional, fnum => 3, rnum => 4, type => bool, + occurrence => optional, opts => []}, + #{name => app, fnum => 4, rnum => 5, type => string, + occurrence => optional, opts => []}, + #{name => namespace, fnum => 5, rnum => 6, + type => string, occurrence => optional, opts => []}]; +find_msg_def('Release') -> + [#{name => version, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => checksum, fnum => 2, rnum => 3, type => bytes, + occurrence => required, opts => []}, + #{name => dependencies, fnum => 3, rnum => 4, + type => {msg, 'Dependency'}, occurrence => repeated, + opts => []}, + #{name => retired, fnum => 4, rnum => 5, + type => {msg, 'RetirementStatus'}, + occurrence => optional, opts => []}]; +find_msg_def('Package') -> + [#{name => releases, fnum => 1, rnum => 2, + type => {msg, 'Release'}, occurrence => repeated, + opts => []}]; +find_msg_def(_) -> error. + + +find_enum_def('RetirementReason') -> + [{'RETIRED_OTHER', 0}, {'RETIRED_INVALID', 1}, + {'RETIRED_SECURITY', 2}, {'RETIRED_DEPRECATED', 3}, + {'RETIRED_RENAMED', 4}]; +find_enum_def(_) -> error. + + +enum_symbol_by_value('RetirementReason', Value) -> + enum_symbol_by_value_RetirementReason(Value). + + +enum_value_by_symbol('RetirementReason', Sym) -> + enum_value_by_symbol_RetirementReason(Sym). + + +enum_symbol_by_value_RetirementReason(0) -> + 'RETIRED_OTHER'; +enum_symbol_by_value_RetirementReason(1) -> + 'RETIRED_INVALID'; +enum_symbol_by_value_RetirementReason(2) -> + 'RETIRED_SECURITY'; +enum_symbol_by_value_RetirementReason(3) -> + 'RETIRED_DEPRECATED'; +enum_symbol_by_value_RetirementReason(4) -> + 'RETIRED_RENAMED'. + + +enum_value_by_symbol_RetirementReason('RETIRED_OTHER') -> + 0; +enum_value_by_symbol_RetirementReason('RETIRED_INVALID') -> + 1; +enum_value_by_symbol_RetirementReason('RETIRED_SECURITY') -> + 2; +enum_value_by_symbol_RetirementReason('RETIRED_DEPRECATED') -> + 3; +enum_value_by_symbol_RetirementReason('RETIRED_RENAMED') -> + 4. + + +get_service_names() -> []. + + +get_service_def(_) -> error. + + +get_rpc_names(_) -> error. + + +find_rpc_def(_, _) -> error. + + + +-spec fetch_rpc_def(_, _) -> no_return(). +fetch_rpc_def(ServiceName, RpcName) -> + erlang:error({no_such_rpc, ServiceName, RpcName}). + + +get_package_name() -> undefined. + + + +gpb_version_as_string() -> + "3.26.4". + +gpb_version_as_list() -> + [3,26,4]. diff --git a/src/hex_pb_signed.erl b/src/hex_pb_signed.erl new file mode 100644 index 000000000..6d8f0d3e3 --- /dev/null +++ b/src/hex_pb_signed.erl @@ -0,0 +1,319 @@ +%% Automatically generated, do not edit +%% Generated by gpb_compile version 3.23.0 +-module(hex_pb_signed). + +-export([encode_msg/2, encode_msg/3]). +-export([decode_msg/2]). +-export([merge_msgs/3]). +-export([verify_msg/2]). +-export([get_msg_defs/0]). +-export([get_msg_names/0]). +-export([get_enum_names/0]). +-export([find_msg_def/1, fetch_msg_def/1]). +-export([find_enum_def/1, fetch_enum_def/1]). +-export([enum_symbol_by_value/2, enum_value_by_symbol/2]). +-export([get_service_names/0]). +-export([get_service_def/1]). +-export([get_rpc_names/1]). +-export([find_rpc_def/2, fetch_rpc_def/2]). +-export([get_package_name/0]). +-export([gpb_version_as_string/0, gpb_version_as_list/0]). + + + +encode_msg(Msg, MsgName) -> + encode_msg(Msg, MsgName, []). + + +encode_msg(Msg, MsgName, _Opts) -> + verify_msg(Msg, MsgName), + case MsgName of 'Signed' -> e_msg_Signed(Msg) end. + + +e_msg_Signed(Msg) -> e_msg_Signed(Msg, <<>>). + + +e_msg_Signed(#{payload := F1} = M, Bin) -> + B1 = begin + TrF1 = id(F1), e_type_bytes(TrF1, <>) + end, + case M of + #{signature := F2} -> + TrF2 = id(F2), e_type_bytes(TrF2, <>); + _ -> B1 + end. + + + +e_type_bytes(Bytes, Bin) -> + Bin2 = e_varint(byte_size(Bytes), Bin), + <>. + +e_varint(N, Bin) when N =< 127 -> <>; +e_varint(N, Bin) -> + Bin2 = <>, + e_varint(N bsr 7, Bin2). + + + +decode_msg(Bin, MsgName) when is_binary(Bin) -> + case MsgName of 'Signed' -> d_msg_Signed(Bin) end. + + + +d_msg_Signed(Bin) -> + dfp_read_field_def_Signed(Bin, 0, 0, id('$undef'), + id('$undef')). + +dfp_read_field_def_Signed(<<10, Rest/binary>>, Z1, Z2, + F1, F2) -> + d_field_Signed_payload(Rest, Z1, Z2, F1, F2); +dfp_read_field_def_Signed(<<18, Rest/binary>>, Z1, Z2, + F1, F2) -> + d_field_Signed_signature(Rest, Z1, Z2, F1, F2); +dfp_read_field_def_Signed(<<>>, 0, 0, F1, F2) -> + S1 = #{payload => F1}, + if F2 == '$undef' -> S1; + true -> S1#{signature => F2} + end; +dfp_read_field_def_Signed(Other, Z1, Z2, F1, F2) -> + dg_read_field_def_Signed(Other, Z1, Z2, F1, F2). + +dg_read_field_def_Signed(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2) + when N < 32 - 7 -> + dg_read_field_def_Signed(Rest, N + 7, X bsl N + Acc, F1, + F2); +dg_read_field_def_Signed(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2) -> + Key = X bsl N + Acc, + case Key of + 10 -> d_field_Signed_payload(Rest, 0, 0, F1, F2); + 18 -> d_field_Signed_signature(Rest, 0, 0, F1, F2); + _ -> + case Key band 7 of + 0 -> skip_varint_Signed(Rest, 0, 0, F1, F2); + 1 -> skip_64_Signed(Rest, 0, 0, F1, F2); + 2 -> skip_length_delimited_Signed(Rest, 0, 0, F1, F2); + 5 -> skip_32_Signed(Rest, 0, 0, F1, F2) + end + end; +dg_read_field_def_Signed(<<>>, 0, 0, F1, F2) -> + S1 = #{payload => F1}, + if F2 == '$undef' -> S1; + true -> S1#{signature => F2} + end. + +d_field_Signed_payload(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2) + when N < 57 -> + d_field_Signed_payload(Rest, N + 7, X bsl N + Acc, F1, + F2); +d_field_Signed_payload(<<0:1, X:7, Rest/binary>>, N, + Acc, _, F2) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Signed(Rest2, 0, 0, NewFValue, F2). + + +d_field_Signed_signature(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2) + when N < 57 -> + d_field_Signed_signature(Rest, N + 7, X bsl N + Acc, F1, + F2); +d_field_Signed_signature(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, _) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Signed(Rest2, 0, 0, F1, NewFValue). + + +skip_varint_Signed(<<1:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2) -> + skip_varint_Signed(Rest, Z1, Z2, F1, F2); +skip_varint_Signed(<<0:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2) -> + dfp_read_field_def_Signed(Rest, Z1, Z2, F1, F2). + + +skip_length_delimited_Signed(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2) + when N < 57 -> + skip_length_delimited_Signed(Rest, N + 7, X bsl N + Acc, + F1, F2); +skip_length_delimited_Signed(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Signed(Rest2, 0, 0, F1, F2). + + +skip_32_Signed(<<_:32, Rest/binary>>, Z1, Z2, F1, F2) -> + dfp_read_field_def_Signed(Rest, Z1, Z2, F1, F2). + + +skip_64_Signed(<<_:64, Rest/binary>>, Z1, Z2, F1, F2) -> + dfp_read_field_def_Signed(Rest, Z1, Z2, F1, F2). + + + + + + +merge_msgs(Prev, New, MsgName) -> + case MsgName of + 'Signed' -> merge_msg_Signed(Prev, New) + end. + +merge_msg_Signed(#{payload := PFpayload} = PMsg, + #{payload := NFpayload} = NMsg) -> + S1 = #{payload => + if NFpayload =:= undefined -> PFpayload; + true -> NFpayload + end}, + case {PMsg, NMsg} of + {_, #{signature := NFsignature}} -> + S1#{signature => NFsignature}; + {#{signature := PFsignature}, _} -> + S1#{signature => PFsignature}; + _ -> S1 + end. + + + +verify_msg(Msg, MsgName) -> + case MsgName of + 'Signed' -> v_msg_Signed(Msg, ['Signed']); + _ -> mk_type_error(not_a_known_message, Msg, []) + end. + + +v_msg_Signed(#{payload := F1} = M, Path) -> + v_type_bytes(F1, [payload | Path]), + case M of + #{signature := F2} -> + v_type_bytes(F2, [signature | Path]); + _ -> ok + end, + ok; +v_msg_Signed(M, Path) when is_map(M) -> + mk_type_error({missing_fields, + [payload] -- maps:keys(M), 'Signed'}, + M, Path); +v_msg_Signed(X, Path) -> + mk_type_error({expected_msg, 'Signed'}, X, Path). + +v_type_bytes(B, _Path) when is_binary(B) -> ok; +v_type_bytes(X, Path) -> + mk_type_error(bad_binary_value, X, Path). + +-spec mk_type_error(_, _, list()) -> no_return(). +mk_type_error(Error, ValueSeen, Path) -> + Path2 = prettify_path(Path), + erlang:error({gpb_type_error, + {Error, [{value, ValueSeen}, {path, Path2}]}}). + + +prettify_path([]) -> top_level; +prettify_path(PathR) -> + list_to_atom(string:join(lists:map(fun atom_to_list/1, + lists:reverse(PathR)), + ".")). + + + +-compile({nowarn_unused_function,id/1}). +-compile({inline,id/1}). +id(X) -> X. + +-compile({nowarn_unused_function,cons/2}). +-compile({inline,cons/2}). +cons(Elem, Acc) -> [Elem | Acc]. + +-compile({nowarn_unused_function,lists_reverse/1}). +-compile({inline,lists_reverse/1}). +'lists_reverse'(L) -> lists:reverse(L). + +-compile({nowarn_unused_function,'erlang_++'/2}). +-compile({inline,'erlang_++'/2}). +'erlang_++'(A, B) -> A ++ B. + + + +get_msg_defs() -> + [{{msg, 'Signed'}, + [#{name => payload, fnum => 1, rnum => 2, type => bytes, + occurrence => required, opts => []}, + #{name => signature, fnum => 2, rnum => 3, + type => bytes, occurrence => optional, opts => []}]}]. + + +get_msg_names() -> ['Signed']. + + +get_enum_names() -> []. + + +fetch_msg_def(MsgName) -> + case find_msg_def(MsgName) of + Fs when is_list(Fs) -> Fs; + error -> erlang:error({no_such_msg, MsgName}) + end. + + +-spec fetch_enum_def(_) -> no_return(). +fetch_enum_def(EnumName) -> + erlang:error({no_such_enum, EnumName}). + + +find_msg_def('Signed') -> + [#{name => payload, fnum => 1, rnum => 2, type => bytes, + occurrence => required, opts => []}, + #{name => signature, fnum => 2, rnum => 3, + type => bytes, occurrence => optional, opts => []}]; +find_msg_def(_) -> error. + + +find_enum_def(_) -> error. + + +-spec enum_symbol_by_value(_, _) -> no_return(). +enum_symbol_by_value(E, V) -> + erlang:error({no_enum_defs, E, V}). + + +-spec enum_value_by_symbol(_, _) -> no_return(). +enum_value_by_symbol(E, V) -> + erlang:error({no_enum_defs, E, V}). + + + +get_service_names() -> []. + + +get_service_def(_) -> error. + + +get_rpc_names(_) -> error. + + +find_rpc_def(_, _) -> error. + + + +-spec fetch_rpc_def(_, _) -> no_return(). +fetch_rpc_def(ServiceName, RpcName) -> + erlang:error({no_such_rpc, ServiceName, RpcName}). + + +get_package_name() -> undefined. + + + +gpb_version_as_string() -> + "3.23.0". + +gpb_version_as_list() -> + [3,23,0]. diff --git a/src/safe_erl_term.xrl b/src/safe_erl_term.xrl new file mode 100644 index 000000000..58a462917 --- /dev/null +++ b/src/safe_erl_term.xrl @@ -0,0 +1,77 @@ +%%% Author : Robert Virding +%%% Purpose : Token definitions for Erlang. + +Definitions. + +D = [0-9] +U = [A-Z] +L = [a-z] +A = ({U}|{L}|{D}|_|@) +WS = ([\000-\s]|%.*) + +Rules. + +{L}{A}* : tokenize_atom(TokenChars, TokenLine). +'(\\\^.|\\.|[^'])*' : tokenize_atom(escape(unquote(TokenChars, TokenLen)), TokenLine). +"(\\\^.|\\.|[^"])*" : {token, {string, TokenLine, escape(unquote(TokenChars, TokenLen))}}. +{D}+ : {token, {integer, TokenLine, list_to_integer(TokenChars)}}. +[\#\[\]}{,+-] : {token, {list_to_atom(TokenChars), TokenLine}}. +(<<|>>|=>) : {token, {list_to_atom(TokenChars), TokenLine}}. +\. : {token, {dot, TokenLine}}. +/ : {token, {'/', TokenLine}}. +{WS}+ : skip_token. + +Erlang code. + +-export([terms/1]). + +terms(Tokens) -> + terms(Tokens, []). + +terms([{dot, _} = H], Buffer) -> + [buffer_to_term([H|Buffer])]; +terms([{dot, _} = H|T], Buffer) -> + [buffer_to_term([H|Buffer])|terms(T, [])]; +terms([H|T], Buffer) -> + terms(T, [H|Buffer]). + +buffer_to_term(Buffer) -> + {ok, Term} = erl_parse:parse_term(lists:reverse(Buffer)), + Term. + +unquote(TokenChars, TokenLen) -> + lists:sublist(TokenChars, 2, TokenLen - 2). + +tokenize_atom(TokenChars, TokenLine) -> + try list_to_existing_atom(TokenChars) of + Atom -> {token, {atom, TokenLine, Atom}} + catch + error:badarg -> {error, "illegal atom " ++ TokenChars} + end. + +escape([$\\|Cs]) -> + do_escape(Cs); +escape([C|Cs]) -> + [C|escape(Cs)]; +escape([]) -> []. + +do_escape([O1,O2,O3|S]) when + O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> + [(O1*8 + O2)*8 + O3 - 73*$0|escape(S)]; +do_escape([$^,C|Cs]) -> + [C band 31|escape(Cs)]; +do_escape([C|Cs]) when C >= $\000, C =< $\s -> + escape(Cs); +do_escape([C|Cs]) -> + [escape_char(C)|escape(Cs)]. + +escape_char($n) -> $\n; %\n = LF +escape_char($r) -> $\r; %\r = CR +escape_char($t) -> $\t; %\t = TAB +escape_char($v) -> $\v; %\v = VT +escape_char($b) -> $\b; %\b = BS +escape_char($f) -> $\f; %\f = FF +escape_char($e) -> $\e; %\e = ESC +escape_char($s) -> $\s; %\s = SPC +escape_char($d) -> $\d; %\d = DEL +escape_char(C) -> C. diff --git a/test/hex/api/validate_cert_test.exs b/test/hex/api/validate_cert_test.exs index ec92a13fd..7f074cc71 100644 --- a/test/hex/api/validate_cert_test.exs +++ b/test/hex/api/validate_cert_test.exs @@ -8,7 +8,7 @@ defmodule Hex.API.ValidateCertTest do end def partial_chain_fun do - &Hex.API.partial_chain([der_encoded_ca_cert()], &1) + &Hex.API.SSL.partial_chain([der_encoded_ca_cert()], &1) end def cert_path(name) do @@ -65,7 +65,7 @@ defmodule Hex.API.ValidateCertTest do :public_key.pkix_path_validation(trusted_cert, cert_path, max_path_length: 20, verify_fun: verify_fun) end - if Hex.API.secure_ssl? do + if Hex.API.SSL.secure_ssl? do test "succeeds to validate normal chain" do assert {:ok, _} = run_validation([server_cert()], :undefined) end @@ -91,7 +91,7 @@ defmodule Hex.API.ValidateCertTest do end test "succeeds to validate with partial chain that is correct" do - partial_chain_fun = &Hex.API.partial_chain(Hex.API.Certs.cacerts, &1) + partial_chain_fun = &Hex.API.SSL.partial_chain(Hex.API.Certs.cacerts, &1) assert {:ok, _} = run_validation([amazon_cert(), verisign_g3(), verisigng5()], partial_chain_fun) end end diff --git a/test/hex/api_test.exs b/test/hex/api_test.exs index 74521fc25..210aefc73 100644 --- a/test/hex/api_test.exs +++ b/test/hex/api_test.exs @@ -3,15 +3,13 @@ defmodule Hex.APITest do @moduletag :integration test "user" do - assert {401, _, _} = Hex.API.User.get("test_user", [key: "something wrong"]) assert {201, _, _} = Hex.API.User.new("test_user", "test_user@mail.com", "hunter42") - - auth = HexTest.HexWeb.new_key([user: "test_user", pass: "hunter42"]) - assert {200, %{"username" => "test_user"}, _} = Hex.API.User.get("test_user", auth) + assert {200, %{"username" => "test_user"}, _} = Hex.API.User.get("test_user") + assert {404, _, _} = Hex.API.User.get("unknown_user") end test "release" do - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) meta = %{name: :pear, app: :pear, version: "0.0.1", build_tools: ["mix"], requirements: [], licenses: ["MIT"], description: "pear"} {tar, _checksum} = Hex.Tar.create(meta, []) @@ -32,7 +30,7 @@ defmodule Hex.APITest do end test "docs" do - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) meta = %{name: :tangerine, app: :tangerine, version: "0.0.1", build_tools: ["mix"], requirements: [], licenses: ["MIT"], description: "tangerine"} {tar, _checksum} = Hex.Tar.create(meta, []) @@ -50,7 +48,7 @@ defmodule Hex.APITest do end test "registry" do - assert {200, _, _} = Hex.API.Registry.get + assert {200, _, _} = Hex.API.Registry.get_package("postgrex") end test "keys" do @@ -62,7 +60,7 @@ defmodule Hex.APITest do assert byte_size(key_b) == 32 auth = [key: key_a] - HexTest.HexWeb.new_package("melon", "0.0.1", %{}, %{}, auth) + HexWeb.new_package("melon", "0.0.1", %{}, %{}, auth) assert {200, body, _} = Hex.API.Key.get(auth) assert Enum.find(body, &(&1["name"] == "key_a")) @@ -93,9 +91,9 @@ defmodule Hex.APITest do end test "owners" do - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) - HexTest.HexWeb.new_package("orange", "0.0.1", %{}, %{}, auth) + HexWeb.new_package("orange", "0.0.1", %{}, %{}, auth) Hex.API.User.new("orange_user", "orange_user@mail.com", "hunter42") assert {200, [%{"username" => "user"}], _} = Hex.API.Package.Owner.get("orange", auth) diff --git a/test/hex/mix_task_test.exs b/test/hex/mix_task_test.exs index 205f6f6c7..840384641 100644 --- a/test/hex/mix_task_test.exs +++ b/test/hex/mix_task_test.exs @@ -230,7 +230,7 @@ defmodule Hex.MixTaskTest do File.write!("mix.lock", ~s(%{"ecto": {:hex, :ecto, "0.2.0"}})) Mix.Task.run "deps.update", ["ecto"] - assert_received {:mix_shell, :info, [" ecto: 0.2.1"]} + assert_received {:mix_shell, :info, ["\e[32m ecto 0.2.1\e[0m"]} end after purge [Ecto.NoConflict.Mixfile, Postgrex.NoConflict.Mixfile, diff --git a/test/hex/mix_test.exs b/test/hex/mix_test.exs index bad634bdc..515c95b47 100644 --- a/test/hex/mix_test.exs +++ b/test/hex/mix_test.exs @@ -2,8 +2,8 @@ defmodule Hex.MixTest do use HexTest.Case, async: true test "from mixlock" do - lock = [ ex_doc: {:hex, :ex_doc, "0.1.0"}, - postgrex: {:hex, :fork, "0.2.1"} ] + lock = [ex_doc: {:hex, :ex_doc, "0.1.0"}, + postgrex: {:hex, :fork, "0.2.1"}] assert Hex.Mix.from_lock(lock) == [{"ex_doc", "ex_doc", "0.1.0"}, {"fork", "postgrex", "0.2.1"}] end diff --git a/test/hex/registry_test.exs b/test/hex/registry_test.exs index 3455bc4fc..e67623464 100644 --- a/test/hex/registry_test.exs +++ b/test/hex/registry_test.exs @@ -1,58 +1,64 @@ defmodule Hex.RegistryTest do use HexTest.Case + alias Hex.Registry.Server - test "stat" do - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) - assert Hex.Registry.stat == {18, 46} - assert Hex.Registry.close == true + defp bypass_csv(versions) do + bypass = Bypass.open + Hex.State.put(:repo, "http://localhost:#{bypass.port}") - # Multiple open and close should yield the same result - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) - assert Hex.Registry.stat == {18, 46} - assert Hex.Registry.close == true + Bypass.expect(bypass, fn %Plug.Conn{request_path: "/installs/hex-1.x.csv"} = conn -> + Plug.Conn.resp(conn, 200, versions_to_csv(versions)) + end) + + bypass end - test "install info, find correct version" do - in_tmp fn -> - Hex.State.put(:registry_updated, false) - Hex.State.put(:home, System.cwd!) + defp versions_to_csv(versions) do + Enum.map_join(versions, "\n", fn {hex, elixir} -> + "#{hex},DIGEST,#{elixir}" + end) + end - path = "registry.ets" - versions = [{"100.0.0", ["100.0.0"]}, {"0.0.1", ["0.0.1"]}, {"99.0.0", ["0.0.1"]}, - {"100.0.0", ["0.0.1"]}, {"98.0.0", ["0.0.1"]}] - create_registry(path, 3, versions, [], []) + test "display new hex version" do + flush() + bypass_csv([{"100.0.0", "1.0.0"}]) + {:ok, server} = Server.start_link(name: false) - Hex.Registry.close - Hex.Utils.ensure_registry!(fetch: false) - assert_received {:mix_shell, :info, ["\e[33mA new Hex version is available (100.0.0), please update with `mix local.hex`\e[0m"]} - end + Server.open(server, registry_path: tmp_path(test_name() <> ".ets")) + Server.close(server) + assert_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} end - test "install info, too new elixir" do - in_tmp fn -> - Hex.State.put(:registry_updated, false) - Hex.State.put(:home, System.cwd!) + test "dont display same hex version" do + flush() + bypass_csv([{"0.0.1", "1.0.0"}]) + {:ok, server} = Server.start_link(name: false) - path = "registry.ets" - versions = [{"100.0.0", ["100.0.0"]}] - create_registry(path, 3, versions, [], []) + Server.open(server, registry_path: tmp_path(test_name() <> ".ets")) + Server.close(server) + refute_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} + end - Hex.Utils.ensure_registry!(fetch: false) - refute_received {:mix_shell, :info, ["A new Hex version is available" <> _]} - end + test "dont display new hex version for too new elixir" do + flush() + bypass_csv([{"100.0.0", "100.0.0"}]) + {:ok, server} = Server.start_link(name: false) + Server.open(server, registry_path: tmp_path(test_name() <> ".ets")) + Server.close(server) + refute_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} end - test "install info, too old hex" do - in_tmp fn -> - Hex.State.put(:registry_updated, false) - Hex.State.put(:home, System.cwd!) + test "only check version once" do + flush() + bypass_csv([{"100.0.0", "1.0.0"}]) + {:ok, server} = Server.start_link(name: false) - path = "registry.ets" - versions = [{"0.0.1", ["0.0.1"]}] - create_registry(path, 3, versions, [], []) + Server.open(server, registry_path: tmp_path(test_name() <> "1.ets")) + Server.close(server) + assert_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} - Hex.Utils.ensure_registry!(fetch: false) - refute_received {:mix_shell, :info, ["A new Hex version is available" <> _]} - end + Server.open(server, registry_path: tmp_path(test_name() <> "2.ets")) + Server.close(server) + refute_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} end end diff --git a/test/hex/repo_test.exs b/test/hex/repo_test.exs index 15936935b..6f379df7a 100644 --- a/test/hex/repo_test.exs +++ b/test/hex/repo_test.exs @@ -9,10 +9,8 @@ defmodule Hex.RepoTest do package_url = Hex.API.repo_url(path) bad_url = Hex.API.repo_url("docs/package") - etag = Hex.Utils.etag(fixture_path("/#{path}")) - - assert {:ok, _} = Hex.Repo.request(package_url, nil) - assert {:ok, _} = Hex.Repo.request(package_url, etag) + assert {:ok, _, nil} = Hex.Repo.request(package_url, nil) + assert {:ok, _, nil} = Hex.Repo.request(package_url, "etag") assert {:error, "Request failed (404)"} = Hex.Repo.request(bad_url, nil) end end diff --git a/test/hex/resolver/backtracks_test.exs b/test/hex/resolver/backtracks_test.exs index a1eac5337..ac8e30175 100644 --- a/test/hex/resolver/backtracks_test.exs +++ b/test/hex/resolver/backtracks_test.exs @@ -3,7 +3,9 @@ defmodule Hex.Resolver.BacktracksTest do import Hex.Resolver.Backtracks, only: [message: 1] setup do - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) + Hex.State.put(:offline?, true) + Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("cache.ets")) + Hex.Registry.prefetch(["foo"]) end test "merge versions" do diff --git a/test/hex/resolver_test.exs b/test/hex/resolver_test.exs index 5603e9eab..03db23933 100644 --- a/test/hex/resolver_test.exs +++ b/test/hex/resolver_test.exs @@ -8,6 +8,11 @@ defmodule Hex.ResolverTest do reqs = reqs(reqs) locked = locked(locked) + [reqs, locked] + |> Enum.concat + |> Enum.map(&elem(&1, 0)) + |> Hex.Registry.prefetch + case Hex.Resolver.resolve(reqs, deps, top_level, locked) do {:ok, dict} -> dict {:error, messages} -> messages <> "\n" @@ -39,7 +44,9 @@ defmodule Hex.ResolverTest do end setup do - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) + Hex.State.put(:offline?, true) + Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("cache.ets")) + :ok end test "simple" do @@ -55,33 +62,25 @@ defmodule Hex.ResolverTest do deps = [bar: nil, foo: "~> 0.3.0"] assert resolve(deps) == """ \e[4mFailed to use "foo" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ deps = [foo: "~> 0.3.0", bar: nil] assert resolve(deps) == """ \e[4mFailed to use \"foo\" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m - - \e[4mFailed to use \"foo\" (version 0.1.0) because\e[0m - \e[1mbar (version 0.1.0)\e[0m requires \e[32m~> 0.1.0\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m - - \e[4mFailed to use \"foo\" (versions 0.2.0 and 0.2.1) because\e[0m - \e[1mbar (version 0.2.0)\e[0m requires \e[32m~> 0.2.0\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ deps = [bar: "~> 0.3.0", foo: nil] assert resolve(deps) == """ \e[4mFailed to use "bar" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ deps = [foo: nil, bar: "~> 0.3.0"] assert resolve(deps) == """ \e[4mFailed to use "bar" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ end @@ -102,26 +101,28 @@ defmodule Hex.ResolverTest do assert resolve(deps) == """ \e[4mFailed to use "decimal" (version 0.1.0) because\e[0m \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m - You specified \e[32m0.1.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[32m0.1.0\e[0m\n\e[0m """ deps = [decimal: "0.1.0", ex_plex: "~> 0.0.2"] assert resolve(deps) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m\n\e[0m + \e[4mFailed to use "decimal" (version 0.1.0) because\e[0m + \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m + \e[1mmix.exs\e[0m specifies \e[32m0.1.0\e[0m\n\e[0m """ deps = [ex_plex: "0.0.2", decimal: nil] assert resolve(deps) == """ \e[4mFailed to use "decimal" (versions 0.0.1 to 0.2.1) because\e[0m \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m - You specified \e[32m>= 0.0.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[32m>= 0.0.0\e[0m\n\e[0m """ deps = [decimal: nil, ex_plex: "0.0.2"] assert resolve(deps) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m\n\e[0m + \e[4mFailed to use "decimal" (versions 0.0.1 to 0.2.1) because\e[0m + \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m + \e[1mmix.exs\e[0m specifies \e[32m>= 0.0.0\e[0m\n\e[0m """ end @@ -176,15 +177,15 @@ defmodule Hex.ResolverTest do assert resolve(deps, locked) == """ \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m - Locked to \e[32m0.0.1\e[0m in your mix.lock\n\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ locked = [decimal: "0.0.1"] deps = [decimal: nil, ex_plex: "0.1.0"] assert resolve(deps, locked) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.1.0)\e[0m requires \e[33m~> 0.1.0\e[0m - Locked to \e[33m0.0.1\e[0m in your mix.lock\n\e[0m + \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m + \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ locked = [decimal: "0.0.1"] @@ -192,15 +193,15 @@ defmodule Hex.ResolverTest do assert resolve(deps, locked) == """ \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m - Locked to \e[32m0.0.1\e[0m in your mix.lock\n\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ locked = [decimal: "0.0.1"] deps = [decimal: "~> 0.0.1", ex_plex: "0.1.0"] assert resolve(deps, locked) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.1.0)\e[0m requires \e[33m~> 0.1.0\e[0m - Locked to \e[33m0.0.1\e[0m in your mix.lock\n\e[0m + \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m + \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ end @@ -208,9 +209,9 @@ defmodule Hex.ResolverTest do deps = [beta: "~> 1.0 and > 1.0.0"] assert resolve(deps) == """ \e[4mFailed to use "beta" because\e[0m - You specified \e[31m~> 1.0 and > 1.0.0\e[0m in your mix.exs * + \e[1mmix.exs\e[0m specifies \e[31m~> 1.0 and > 1.0.0\e[0m * \e[0m - * This requirement failed because by default pre-releases are never matched. To match against pre-releases include a pre-release in the requirement string: "~> 2.0-beta".\n + * This requirement does not match pre-releases. To match pre-releases include a pre-release in the requirement, such as: \"~> 2.0-beta\".\n """ deps = [beta: "~> 1.0-beta and > 1.0.0-beta"] @@ -220,9 +221,9 @@ defmodule Hex.ResolverTest do test "only mix.exs conflicts" do deps = [decimal: "~> 0.0.1", ex_plex: "0.2.0"] assert resolve(deps, []) == """ - \e[4mFailed to use "decimal" (versions 0.2.0 and 0.2.1) because\e[0m - \e[1mex_plex (version 0.2.0)\e[0m requires \e[32m~> 0.2.0\e[0m - You specified \e[31m~> 0.0.1\e[0m in your mix.exs\n\e[0m + \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m + \e[1mex_plex (version 0.2.0)\e[0m requires \e[31m~> 0.2.0\e[0m + \e[1mmix.exs\e[0m specifies \e[32m~> 0.0.1\e[0m\n\e[0m """ end diff --git a/test/hex/scm_test.exs b/test/hex/scm_test.exs new file mode 100644 index 000000000..6eb6258b0 --- /dev/null +++ b/test/hex/scm_test.exs @@ -0,0 +1,15 @@ +defmodule Hex.SCMTest do + use ExUnit.Case, async: true + + test "guess build_tools" do + empty_meta = %{} + guessed_meta = %{"build_tools" => ["mix"]} + no_tools_meta = %{"files" => ["README.md"]} + tools_meta = %{"files" => ["README.md", "mix.exs", "lib"]} + + assert [] = Hex.SCM.guess_build_tools(empty_meta) + assert ["mix"] = Hex.SCM.guess_build_tools(guessed_meta) + assert [] = Hex.SCM.guess_build_tools(no_tools_meta) + assert ["mix"] = Hex.SCM.guess_build_tools(tools_meta) + end +end diff --git a/test/mix/tasks/hex/docs_test.exs b/test/mix/tasks/hex/docs_test.exs index 66a7e85cd..fe2c6917b 100644 --- a/test/mix/tasks/hex/docs_test.exs +++ b/test/mix/tasks/hex/docs_test.exs @@ -3,12 +3,12 @@ defmodule Mix.Tasks.Hex.DocsTest do @moduletag :integration test "open fails when docs not found" do - docs_home = :home |> Hex.State.fetch!() |> Path.join("docs") + docs_home = Path.join(Hex.State.fetch!(:home), "docs") package = "decimal" version = "1.1.2" message = "Documentation file not found: #{docs_home}/#{package}/#{version}/index.html" assert_raise Mix.Error, message, fn -> - Mix.Tasks.Hex.Docs.run(["open", package, version]) + Mix.Tasks.Hex.Docs.run(["open", package, version, "--offline"]) end end @@ -18,12 +18,11 @@ defmodule Mix.Tasks.Hex.DocsTest do latest_version = "1.1.2" bypass_mirror() Hex.State.put(:home, tmp_path()) + docs_home = Path.join(Hex.State.fetch!(:home), "docs") - docs_home = :home |> Hex.State.fetch!() |> Path.join("docs") - - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) - HexTest.HexWeb.new_package(package, old_version, %{}, %{}, auth) - HexTest.HexWeb.new_package(package, latest_version, %{}, %{}, auth) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) + HexWeb.new_package(package, old_version, %{}, %{}, auth) + HexWeb.new_package(package, latest_version, %{}, %{}, auth) in_tmp "docs", fn -> Mix.Tasks.Hex.Docs.run(["fetch", package]) @@ -42,7 +41,10 @@ defmodule Mix.Tasks.Hex.DocsTest do bypass_mirror() Hex.State.put(:home, tmp_path()) - docs_home = :home |> Hex.State.fetch!() |> Path.join("docs") + docs_home = + :home + |> Hex.State.fetch!() + |> Path.join("docs") in_tmp "docs", fn -> Mix.Tasks.Hex.Docs.run(["fetch", package, version]) @@ -65,18 +67,13 @@ defmodule Mix.Tasks.Hex.DocsTest do end test "invalid arguments for docs task" do - deprecation_msg = """ - [deprecation] Calling mix hex.docs without a command is deprecated, please use: - mix hex.publish docs - """ - assert_raise Mix.Error, deprecation_msg, fn -> - Mix.Tasks.Hex.Docs.run([]) - end + exception = assert_raise Mix.Error, fn -> Mix.Tasks.Hex.Docs.run([]) end + assert Exception.message(exception) =~ ~s([deprecation] The "mix hex.docs" command has changed) invalid_args_msg = """ - invalid arguments, expected one of: - mix hex.docs fetch PACKAGE [VERSION] - mix hex.docs open PACKAGE [VERSION] + Invalid arguments, expected one of: + mix hex.docs fetch PACKAGE [VERSION] + mix hex.docs open PACKAGE [VERSION] """ assert_raise Mix.Error, invalid_args_msg, fn -> @@ -91,7 +88,7 @@ defmodule Mix.Tasks.Hex.DocsTest do end assert_raise Mix.Error, msg, fn -> - Mix.Tasks.Hex.Docs.run(["open"]) + Mix.Tasks.Hex.Docs.run(["open", "--offline"]) end end end diff --git a/test/mix/tasks/hex/info_test.exs b/test/mix/tasks/hex/info_test.exs index 467519ed4..029759baa 100644 --- a/test/mix/tasks/hex/info_test.exs +++ b/test/mix/tasks/hex/info_test.exs @@ -4,7 +4,7 @@ defmodule Mix.Tasks.Hex.InfoTest do test "package" do Mix.Tasks.Hex.Info.run(["ex_doc"]) - assert_received {:mix_shell, :info, ["Builds docs\n"]} + assert_received {:mix_shell, :info, ["Some description\n"]} assert_received {:mix_shell, :info, ["Config: {:ex_doc, \"~> 0.1.0\"}"]} assert_received {:mix_shell, :info, ["Maintainers: John Doe, Jane Doe"]} @@ -22,22 +22,4 @@ defmodule Mix.Tasks.Hex.InfoTest do Mix.Tasks.Hex.Info.run(["ex_doc", "1.2.3"]) assert_received {:mix_shell, :error, ["No release with name ex_doc 1.2.3"]} end - - test "general" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - assert {200, data, _} = Hex.API.Registry.get - File.write!(Hex.Registry.ETS.path <> ".gz", data) - File.write!(Hex.Registry.ETS.path, :zlib.gunzip(data)) - Mix.Tasks.Hex.Info.run([]) - - message = "Hex: " <> Hex.version - assert_received {:mix_shell, :info, [^message]} - assert_received {:mix_shell, :info, ["Registry file available (last updated: " <> _]} - assert_received {:mix_shell, :info, ["File size: " <> _]} - assert_received {:mix_shell, :info, ["Packages #: " <> _]} - assert_received {:mix_shell, :info, ["Versions #: " <> _]} - end - end end diff --git a/test/mix/tasks/hex/key_test.exs b/test/mix/tasks/hex/key_test.exs index 1711e4ba2..f46716d79 100644 --- a/test/mix/tasks/hex/key_test.exs +++ b/test/mix/tasks/hex/key_test.exs @@ -1,77 +1,14 @@ defmodule Mix.Tasks.Hex.KeyTest do use HexTest.Case - @moduletag :integration - test "list keys" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) + test "mix hex.key task is deprecated" do + deprecation_msg = """ + [deprecation] The mix hex.key task is deprecated, please use: + mix hex.user + """ - auth = HexTest.HexWeb.new_user("list_keys", "list_keys@mail.com", "password", "list_keys") - Hex.Config.update(auth) - - assert {200, [%{"name" => "list_keys"}], _} = Hex.API.Key.get(auth) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["list"]) - assert_received {:mix_shell, :info, ["list_keys" <> _]} - end - end - - test "remove key" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - auth_a = HexTest.HexWeb.new_user("remove_key", "remove_key@mail.com", "password", "remove_key_a") - auth_b = HexTest.HexWeb.new_key("remove_key", "password", "remove_key_b") - Hex.Config.update(auth_a) - - assert {200, _, _} = Hex.API.Key.get(auth_a) - assert {200, _, _} = Hex.API.Key.get(auth_b) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["remove", "remove_key_b"]) - assert_received {:mix_shell, :info, ["Removing key remove_key_b..."]} - - assert {200, _, _} = Hex.API.Key.get(auth_a) - assert {401, _, _} = Hex.API.Key.get(auth_b) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["remove", "remove_key_a"]) - assert_received {:mix_shell, :info, ["Removing key remove_key_a..."]} - assert_received {:mix_shell, :info, ["User `remove_key` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} - - assert {401, _, _} = Hex.API.Key.get(auth_a) - - config = Hex.Config.read - refute config[:username] - refute config[:key] - refute config[:encrypted_key] - end - end - - test "remove all keys" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - auth_a = HexTest.HexWeb.new_user("remove_all_keys", "remove_all_keys@mail.com", "password", "remove_all_keys_a") - auth_b = HexTest.HexWeb.new_key("remove_all_keys", "password", "remove_all_keys_b") - Hex.Config.update(auth_a) - - assert {200, _, _} = Hex.API.Key.get(auth_a) - assert {200, _, _} = Hex.API.Key.get(auth_b) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["remove", "--all"]) - assert_received {:mix_shell, :info, ["Removing all keys..."]} - assert_received {:mix_shell, :info, ["User `remove_all_keys` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} - - assert {401, _, _} = Hex.API.Key.get(auth_a) - assert {401, _, _} = Hex.API.Key.get(auth_b) - - config = Hex.Config.read - refute config[:username] - refute config[:key] - refute config[:encrypted_key] + assert_raise Mix.Error, deprecation_msg, fn -> + Mix.Tasks.Hex.Key.run([""]) end end end diff --git a/test/mix/tasks/hex/outdated_test.exs b/test/mix/tasks/hex/outdated_test.exs index 91cca29b0..8e99d2494 100644 --- a/test/mix/tasks/hex/outdated_test.exs +++ b/test/mix/tasks/hex/outdated_test.exs @@ -45,6 +45,15 @@ defmodule Mix.Tasks.Hex.OutdatedTest do end end + defmodule OutdatedMultiDeps.Mixfile do + def project do + [app: :outdated_app, + version: "0.0.2", + deps: [{:baz, "0.1.0"}, + {:bar, "0.1.0"}]] + end + end + test "outdated" do Mix.Project.push OutdatedDeps.Mixfile @@ -57,7 +66,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated" - bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "0.1.0", :reset, " "] + bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string @@ -78,15 +87,15 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", ["--all"] - bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "0.1.0", :reset, " "] + bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string - foo = [:bright, "foo", :reset, " ", "0.1.0", :reset, " ", :red, "0.1.1", :reset, " ", :green, "~> 0.1.0", :reset, " "] + foo = [:bright, "foo", :reset, " ", "0.1.0", :reset, " ", :red, "0.1.1", :reset, " ", :green, "Yes", :reset, " "] |> IO.ANSI.format |> List.to_string - ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "~> 0.0.1", :reset, " "] + ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "No", :reset, " "] |> IO.ANSI.format |> List.to_string @@ -96,6 +105,30 @@ defmodule Mix.Tasks.Hex.OutdatedTest do end end + test "outdated --all with multiple dependent packages" do + Mix.Project.push OutdatedMultiDeps.Mixfile + + in_tmp fn -> + Hex.State.put(:home, tmp_path()) + Mix.Dep.Lock.write %{ + foo: {:hex, :foo, "0.1.0"}, + bar: {:hex, :bar, "0.1.0"}, + baz: {:hex, :baz, "0.1.0"} + } + + Mix.Task.run "deps.get" + flush() + + Mix.Task.run "hex.outdated", ["--all"] + + foo = [:bright, "foo", :reset, " ", "0.1.0", :reset, " ", :red, "0.1.1", :reset, " ", :red, "No", :reset, " "] + |> IO.ANSI.format + |> List.to_string + + assert_received {:mix_shell, :info, [^foo]} + end + end + test "outdated --pre" do Mix.Project.push OutdatedBetaDeps.Mixfile @@ -108,7 +141,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", [] - beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :green, "1.0.0", :reset, " ", :green, ">= 0.0.0", :reset, " "] + beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :green, "1.0.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string assert_received {:mix_shell, :info, [^beta]} @@ -116,7 +149,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.reenable "hex.outdated" Mix.Task.run "hex.outdated", ["--pre"] - beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :red, "1.1.0-beta", :reset, " ", :red, ">= 0.0.0", :reset, " "] + beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :red, "1.1.0-beta", :reset, " ", :red, "No", :reset, " "] |> IO.ANSI.format |> List.to_string assert_received {:mix_shell, :info, [^beta]} @@ -135,7 +168,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", ["ex_doc"] msg = ["There is newer version of the dependency available ", :bright, "0.1.0 > 0.0.1", :reset, "!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> List.to_string assert_received {:mix_shell, :info, [^msg]} @@ -168,7 +201,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", ["ex_doc"] msg = ["Current version ", :bright, "0.1.0", :reset, " of dependency is up to date!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> List.to_string assert_received {:mix_shell, :info, [^msg]} end @@ -227,11 +260,11 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "deps.get" flush() - ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "~> 0.0.1", :reset, " "] + ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "No", :reset, " "] |> IO.ANSI.format |> List.to_string - bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "0.1.0", :reset, " "] + bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string diff --git a/test/mix/tasks/hex/owner_test.exs b/test/mix/tasks/hex/owner_test.exs index feabccee3..3ed302c6c 100644 --- a/test/mix/tasks/hex/owner_test.exs +++ b/test/mix/tasks/hex/owner_test.exs @@ -3,46 +3,46 @@ defmodule Mix.Tasks.Hex.OwnerTest do @moduletag :integration test "add owner" do - auth1 = HexTest.HexWeb.new_user("owner_user1", "owner_user1@mail.com", "pass", "key") - auth2 = HexTest.HexWeb.new_user("owner_user2", "owner_user2@mail.com", "pass", "key") - HexTest.HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth1) + auth = HexWeb.new_user("owner_user1", "owner_user1@mail.com", "passpass", "key") + HexWeb.new_user("owner_user2", "owner_user2@mail.com", "passpass", "key") + HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) - Hex.Config.update(auth1) + Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["add", "owner_package1", "owner_user2@mail.com"]) assert_received {:mix_shell, :info, ["Adding owner owner_user2@mail.com to owner_package1"]} - assert {200, %{"owned_packages" => %{"owner_package1" => _}}, _} = Hex.API.User.get("owner_user2", auth2) + assert {200, %{"owned_packages" => %{"owner_package1" => _}}, _} = Hex.API.User.get("owner_user2") end test "remove owner" do - auth = HexTest.HexWeb.new_user("owner_user3", "owner_user3@mail.com", "pass", "key") - HexTest.HexWeb.new_user("owner_user4", "owner_user4@mail.com", "pass", "key") - HexTest.HexWeb.new_package("owner_package2", "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user3", "owner_user3@mail.com", "passpass", "key") + HexWeb.new_user("owner_user4", "owner_user4@mail.com", "passpass", "key") + HexWeb.new_package("owner_package2", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["add", "owner_package2", "owner_user4@mail.com"]) Mix.Tasks.Hex.Owner.run(["remove", "owner_package2", "owner_user3@mail.com"]) assert_received {:mix_shell, :info, ["Removing owner owner_user3@mail.com from owner_package2"]} - assert {200, %{"owned_packages" => owned}, _} = Hex.API.User.get("owner_user3", auth) + assert {200, %{"owned_packages" => owned}, _} = Hex.API.User.get("owner_user3") assert owned == %{} end test "list owners" do - auth = HexTest.HexWeb.new_user("owner_user5", "owner_user5@mail.com", "pass", "key") - HexTest.HexWeb.new_package("owner_package3", "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user5", "owner_user5@mail.com", "passpass", "key") + HexWeb.new_package("owner_package3", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["list", "owner_package3"]) assert_received {:mix_shell, :info, ["owner_user5@mail.com"]} end @@ -51,22 +51,17 @@ defmodule Mix.Tasks.Hex.OwnerTest do package1 = "owner_package4" package2 = "owner_package5" owner_email = "owner_user6@mail.com" - auth = HexTest.HexWeb.new_user("owner_user6", owner_email, "pass", "key") - HexTest.HexWeb.new_package(package1, "0.0.1", [], %{}, auth) - HexTest.HexWeb.new_package(package2, "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user6", owner_email, "passpass", "key") + HexWeb.new_package(package1, "0.0.1", [], %{}, auth) + HexWeb.new_package(package2, "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} - send self(), {:mix_shell_input, :prompt, "pass"} - send self(), {:mix_shell_input, :prompt, "pass"} - Mix.Tasks.Hex.Owner.run(["add", package1, owner_email]) - Mix.Tasks.Hex.Owner.run(["add", package2, owner_email]) - + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["packages"]) - owner_package4_msg = "#{package1} - http://localhost:4043/packages/#{package1}" - owner_package5_msg = "#{package2} - http://localhost:4043/packages/#{package2}" + owner_package4_msg = "#{package1} - https://hex.pm/packages/#{package1}" + owner_package5_msg = "#{package2} - https://hex.pm/packages/#{package2}" assert_received {:mix_shell, :info, [^owner_package4_msg]} assert_received {:mix_shell, :info, [^owner_package5_msg]} end diff --git a/test/mix/tasks/hex/public_keys_test.exs b/test/mix/tasks/hex/public_keys_test.exs index 087e98fc6..2a6fb7488 100644 --- a/test/mix/tasks/hex/public_keys_test.exs +++ b/test/mix/tasks/hex/public_keys_test.exs @@ -46,32 +46,32 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do test "list default keys" do Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} end test "add and remove keys" do in_tmp fn -> Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", "other.repo", "my_key.pem", "--force"]) + Mix.Tasks.Hex.PublicKeys.run(["add", "http://other.repo", "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() + verify!() Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} - assert_received {:mix_shell, :info, ["* other.repo"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} + assert_received {:mix_shell, :info, ["* http://other.repo"]} - Mix.Tasks.Hex.PublicKeys.run(["remove", "other.repo"]) + Mix.Tasks.Hex.PublicKeys.run(["remove", "http://other.repo"]) Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} - refute_received {:mix_shell, :info, ["* other.repo"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} + refute_received {:mix_shell, :info, ["* http://other.repo"]} end end test "fails to verify with wrong key" do Hex.State.put(:hexpm_pk, @public_key) assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() + verify!() end end @@ -79,13 +79,13 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do in_tmp fn -> Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", "hex.pm", "my_key.pem", "--force"]) + Mix.Tasks.Hex.PublicKeys.run(["add", "https://repo.hex.pm", "my_key.pem", "--force"]) Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() + verify!() end end end @@ -93,90 +93,46 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do test "fails when no public key is stored" do Hex.State.put(:repo, Hex.State.fetch!(:mirror)) assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() - end - end - - test "fetch signature from x-hex-signature header" do - bypass = bypass_registry_with_header("x-hex-signature") - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) - - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() - end - end - - test "fetch signature from x-amz-meta-signature header" do - bypass = bypass_registry_with_header("x-amz-meta-signature") - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) - - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() + verify!() end end test "fetch signature from file" do - bypass = bypass_registry_with_file() - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) + bypass_registry() + repo = Hex.State.fetch!(:repo) in_tmp fn -> Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", @public_key) Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() + verify!() end end test "fails to verify signature from file" do - bypass = bypass_registry_with_file() - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) + bypass_registry() + repo = Hex.State.fetch!(:repo) in_tmp fn -> Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", Hex.State.fetch!(:hexpm_pk)) Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() + verify!() end end end - defp bypass_registry_with_header(header) do - bypass = bypass_setup() - - Bypass.expect bypass, fn %Plug.Conn{request_path: "/registry.ets.gz"} = conn -> - registry = File.read!(tmp_path("registry.ets")) |> :zlib.gzip - signature = sign(registry) - - conn - |> Plug.Conn.put_resp_header(header, signature) - |> Plug.Conn.resp(200, registry) - end - - bypass - end - - defp bypass_registry_with_file do + defp bypass_registry do bypass = bypass_setup() Bypass.expect bypass, fn - %Plug.Conn{request_path: "/registry.ets.gz"} = conn -> - registry = File.read!(tmp_path("registry.ets")) |> :zlib.gzip - Plug.Conn.resp(conn, 200, registry) - %Plug.Conn{request_path: "/registry.ets.gz.signed"} = conn -> - registry = File.read!(tmp_path("registry.ets")) |> :zlib.gzip - signature = sign(registry) - Plug.Conn.resp(conn, 200, signature) + %Plug.Conn{request_path: "/packages/postgrex"} = conn -> + file = + %{payload: "foobar", signature: sign("foobar")} + |> :hex_pb_signed.encode_msg(:Signed) + |> :zlib.gzip + Plug.Conn.resp(conn, 200, file) end bypass @@ -188,11 +144,17 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do bypass end - defp sign(registry) do - [entry | _ ] = :public_key.pem_decode(@private_key) + defp sign(file) do + [entry | _] = :public_key.pem_decode(@private_key) key = :public_key.pem_entry_decode(entry) - :public_key.sign(registry, :sha512, key) - |> Base.encode16(case: :lower) + :public_key.sign(file, :sha512, key) + end + + defp verify! do + {200, body, _headers} = Hex.API.Registry.get_package("postgrex") + body + |> :zlib.gunzip + |> Hex.API.Registry.verify end end diff --git a/test/mix/tasks/hex/publish_test.exs b/test/mix/tasks/hex/publish_test.exs index ca5fb6c67..c94c279cb 100644 --- a/test/mix/tasks/hex/publish_test.exs +++ b/test/mix/tasks/hex/publish_test.exs @@ -44,7 +44,7 @@ defmodule Mix.Tasks.Hex.PublishTest do Mix.Tasks.Hex.Publish.run(["package", "--no-progress"]) assert {200, _, _} = Hex.API.Release.get("release_a", "0.0.1") - msg = "Before publishing, please read Hex Code of Conduct: https://hex.pm/policies/codeofconduct" + msg = "Before publishing, please read the Code of Conduct: https://hex.pm/policies/codeofconduct" assert_received {:mix_shell, :info, [^msg]} send self(), {:mix_shell_input, :yes?, true} @@ -98,11 +98,11 @@ defmodule Mix.Tasks.Hex.PublishTest do setup_auth("user", "hunter42") raised_message = """ - invalid arguments, expected one of: - mix hex.publish - mix hex.publish package - mix hex.publish docs - """ + Invalid arguments, expected one of: + mix hex.publish + mix hex.publish package + mix hex.publish docs + """ send self(), {:mix_shell_input, :prompt, "hunter42"} assert_raise Mix.Error, raised_message, fn -> diff --git a/test/mix/tasks/hex/registry_test.exs b/test/mix/tasks/hex/registry_test.exs deleted file mode 100644 index 790a2e9e4..000000000 --- a/test/mix/tasks/hex/registry_test.exs +++ /dev/null @@ -1,27 +0,0 @@ -defmodule Mix.Tasks.Hex.RegistryTest do - use HexTest.Case - alias Mix.Tasks.Hex.Registry - - test "dump" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - File.write!("registry.ets.gz", "ETS") - dest = Path.expand("dest.ets.gz") - Registry.run(["dump", dest]) - assert File.read!(dest) == "ETS" - end - end - - test "load" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - source = Path.expand("source.ets.gz") - File.write!(source, :zlib.gzip("ETS")) - Registry.run(["load", source]) - path = Hex.Registry.ETS.path - path_gz = path <> ".gz" - assert File.read!(path) == "ETS" - assert File.regular?(path_gz) - end - end -end diff --git a/test/mix/tasks/hex/retire_test.exs b/test/mix/tasks/hex/retire_test.exs new file mode 100644 index 000000000..8f913aada --- /dev/null +++ b/test/mix/tasks/hex/retire_test.exs @@ -0,0 +1,24 @@ +defmodule Mix.Tasks.Hex.RetireTest do + use HexTest.Case + @moduletag :integration + + test "retire and unretire package" do + auth = HexWeb.new_user("retire_user", "retire_user@mail.com", "passpass", "key") + HexWeb.new_package("retire_package", "0.0.1", [], %{}, auth) + + Hex.State.put(:home, tmp_path()) + Hex.Config.update(auth) + + send self(), {:mix_shell_input, :prompt, "passpass"} + Mix.Tasks.Hex.Retire.run(["retire_package", "0.0.1", "renamed", "--message", "message"]) + + assert {200, %{"retirement" => %{"message" => "message", "reason" => "renamed"}}, _} = + Hex.API.Release.get("retire_package", "0.0.1") + + send self(), {:mix_shell_input, :prompt, "passpass"} + Mix.Tasks.Hex.Retire.run(["retire_package", "0.0.1", "--unretire"]) + + assert {200, %{"retirement" => nil}, _} = + Hex.API.Release.get("retire_package", "0.0.1") + end +end diff --git a/test/mix/tasks/hex/search_test.exs b/test/mix/tasks/hex/search_test.exs index 5cbadbfda..1dfb6d97c 100644 --- a/test/mix/tasks/hex/search_test.exs +++ b/test/mix/tasks/hex/search_test.exs @@ -2,16 +2,10 @@ defmodule Mix.Tasks.Hex.SearchTest do use HexTest.Case @moduletag :integration - setup do - Hex.State.put(:registry_updated, true) - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) - end - test "search" do - Mix.Tasks.Hex.Search.run(["ex"]) - assert_received {:mix_shell, :info, ["ex_doc 0.1.0"]} - assert_received {:mix_shell, :info, ["ex_plex 0.2.0"]} - assert_received {:mix_shell, :info, ["postgrex 0.2.1"]} + Mix.Tasks.Hex.Search.run(["doc"]) + assert_received {:mix_shell, :info, ["ex_doc\e[0m 0.1.0\e[0m https://hex.pm/packages/ex_doc" <> _]} + assert_received {:mix_shell, :info, ["only_doc\e[0m 0.1.0\e[0m https://hex.pm/packages/only_doc" <> _]} end test "empty search" do diff --git a/test/mix/tasks/hex/user_test.exs b/test/mix/tasks/hex/user_test.exs index 463a38e21..0191b6e6e 100644 --- a/test/mix/tasks/hex/user_test.exs +++ b/test/mix/tasks/hex/user_test.exs @@ -21,9 +21,11 @@ defmodule Mix.Tasks.Hex.UserTest do Mix.Tasks.Hex.User.run(["register"]) - auth = get_auth("eric", "hunter42") - assert {200, body, _} = Hex.API.User.get("eric", auth) - assert body["email"] == "mail@mail.com" + assert {200, body, _} = Hex.API.User.get("eric") + assert body["username"] == "eric" + # TODO: re-enable after grace period + # or test using a different authenticated endpoint + # assert body["email"] == "mail@mail.com" end test "auth" do @@ -140,4 +142,77 @@ defmodule Mix.Tasks.Hex.UserTest do assert_received {:mix_shell, :info, ["ausername"]} end end + + test "list keys" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + auth = HexWeb.new_user("list_keys", "list_keys@mail.com", "password", "list_keys") + Hex.Config.update(auth) + + assert {200, [%{"name" => "list_keys"}], _} = Hex.API.Key.get(auth) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--list"]) + assert_received {:mix_shell, :info, ["list_keys" <> _]} + end + end + + test "remove key" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + auth_a = HexWeb.new_user("remove_key", "remove_key@mail.com", "password", "remove_key_a") + auth_b = HexWeb.new_key("remove_key", "password", "remove_key_b") + Hex.Config.update(auth_a) + + assert {200, _, _} = Hex.API.Key.get(auth_a) + assert {200, _, _} = Hex.API.Key.get(auth_b) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--remove", "remove_key_b"]) + assert_received {:mix_shell, :info, ["Removing key remove_key_b..."]} + + assert {200, _, _} = Hex.API.Key.get(auth_a) + assert {401, _, _} = Hex.API.Key.get(auth_b) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--remove", "remove_key_a"]) + assert_received {:mix_shell, :info, ["Removing key remove_key_a..."]} + assert_received {:mix_shell, :info, ["User `remove_key` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} + + assert {401, _, _} = Hex.API.Key.get(auth_a) + + config = Hex.Config.read + refute config[:username] + refute config[:key] + refute config[:encrypted_key] + end + end + + test "remove all keys" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + auth_a = HexWeb.new_user("remove_all_keys", "remove_all_keys@mail.com", "password", "remove_all_keys_a") + auth_b = HexWeb.new_key("remove_all_keys", "password", "remove_all_keys_b") + Hex.Config.update(auth_a) + + assert {200, _, _} = Hex.API.Key.get(auth_a) + assert {200, _, _} = Hex.API.Key.get(auth_b) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--remove-all"]) + assert_received {:mix_shell, :info, ["Removing all keys..."]} + assert_received {:mix_shell, :info, ["User `remove_all_keys` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} + + assert {401, _, _} = Hex.API.Key.get(auth_a) + assert {401, _, _} = Hex.API.Key.get(auth_b) + + config = Hex.Config.read + refute config[:username] + refute config[:key] + refute config[:encrypted_key] + end + end end diff --git a/test/support/case.ex b/test/support/case.ex index fccc46a6f..cad2f03ab 100644 --- a/test/support/case.ex +++ b/test/support/case.ex @@ -5,7 +5,7 @@ defmodule HexTest.Case do quote do import unquote(__MODULE__) alias HexTest.Case - # alias HexTest.HexWeb + alias HexTest.HexWeb end end @@ -37,6 +37,10 @@ defmodule HexTest.Case do Path.join(fixture_path(), extension) end + defmacro test_name do + Path.join(["#{__CALLER__.module}", "#{elem(__CALLER__.function, 0)}"]) + end + defmacro in_tmp(fun) do path = Path.join(["#{__CALLER__.module}", "#{elem(__CALLER__.function, 0)}"]) quote do @@ -76,97 +80,88 @@ defmodule HexTest.Case do end end - @ets_table :hex_ets_registry - @version 4 + @ets_table :hex_index def create_test_registry(path) do - packages = + versions = Enum.reduce(test_registry(), %{}, fn {name, vsn, _}, dict -> - Map.update(dict, "#{name}", [vsn], &[vsn|&1]) + Map.update(dict, Atom.to_string(name), [vsn], &(&1 ++ [vsn])) end) + |> Enum.to_list - packages = - Enum.map(packages, fn {name, vsns} -> - {"#{name}", [Enum.sort(vsns, &(Version.compare(&1, &2) == :lt))]} - end) - - releases = - Enum.map(test_registry(), fn {name, version, deps} -> + deps = + Enum.map(test_registry(), fn {name, vsn, deps} -> deps = Enum.map(deps, fn - {name, req} -> ["#{name}", req, false, "#{name}"] - {name, req, optional} -> ["#{name}", req, optional, "#{name}"] - {name, req, optional, app} -> ["#{name}", req, optional, "#{app}"] + {name, req} -> + {Atom.to_string(name), Atom.to_string(name), req, false} + {name, req, optional} -> + {Atom.to_string(name), Atom.to_string(name), req, optional} + {name, req, optional, app} -> + {Atom.to_string(name), Atom.to_string(app), req, optional} end) - {{"#{name}", version}, [deps, nil]} + {{Atom.to_string(name), vsn}, deps} end) - create_registry(path, @version, [], releases, packages) + create_registry(path, versions, deps) end - def create_registry(path, version, installs, releases, packages) do + defp create_registry(path, versions, deps) do tid = :ets.new(@ets_table, []) - :ets.insert(tid, {:"$$version$$", version}) - :ets.insert(tid, {:"$$installs2$$", installs}) - :ets.insert(tid, releases ++ packages) - :ok = :ets.tab2file(tid, String.to_char_list(path)) + versions = Enum.map(versions, fn {pkg, val} -> {{:versions, pkg}, val} end) + deps = Enum.map(deps, fn {{pkg, vsn}, val} -> {{:deps, pkg, vsn}, val} end) + :ets.insert(tid, versions ++ deps) + :ok = :ets.tab2file(tid, Hex.string_to_charlist(path)) :ets.delete(tid) end + # Needs to be sorted on names and versions defp test_registry do - [ {:foo, "0.0.1", []}, - {:foo, "0.1.0", []}, - {:foo, "0.2.0", []}, - {:foo, "0.2.1", []}, - {:bar, "0.0.1", []}, - {:bar, "0.1.0", [foo: "~> 0.1.0"]}, - {:bar, "0.2.0", [foo: "~> 0.2.0"]}, - - {:decimal, "0.0.1", []}, - {:decimal, "0.1.0", []}, - {:decimal, "0.2.0", []}, - {:decimal, "0.2.1", []}, - {:ex_plex, "0.0.1", []}, - {:ex_plex, "0.0.2", [decimal: "0.1.1"]}, - {:ex_plex, "0.1.0", [decimal: "~> 0.1.0"]}, - {:ex_plex, "0.1.2", [decimal: "~> 0.1.0"]}, - {:ex_plex, "0.2.0", [decimal: "~> 0.2.0"]}, - - {:jose, "0.2.0", []}, - {:jose, "0.2.1", []}, - {:eric, "0.0.1", []}, - {:eric, "0.0.2", []}, - {:eric, "0.1.0", [jose: "~> 0.1.0"]}, - {:eric, "0.1.2", [jose: "~> 0.1.0"]}, - {:eric, "0.2.0", [jose: "~> 0.3.0"]}, - - {:ex_doc, "0.0.1", []}, - {:ex_doc, "0.0.2", []}, - {:ex_doc, "0.1.0", []}, - {:postgrex, "0.2.0", [ex_doc: "0.0.1"]}, - {:postgrex, "0.2.1", [ex_doc: "~> 0.1.0"]}, - {:ecto, "0.2.0", [postgrex: "~> 0.2.0", ex_doc: "~> 0.0.1"]}, - {:ecto, "0.2.1", [postgrex: "~> 0.2.1", ex_doc: "0.1.0"]}, - {:phoenix, "0.0.1", [postgrex: "~> 0.2"]}, - - {:only_doc, "0.1.0", [{:ex_doc, ">= 0.0.0", true}]}, - - {:has_optional, "0.1.0", [{:ex_doc, "~> 0.0.1", true}]}, - - {:package_name, "0.1.0", []}, - {:depend_name, "0.2.0", [{:package_name, ">= 0.0.0", false, :app_name}]}, - - {:poison, "1.5.2", []}, - {:poison, "2.0.0", []}, - {:phoenix, "1.1.3", [poison: "~> 1.5 or ~> 2.0"]}, - {:phoenix, "1.1.2", [poison: "~> 1.5 or ~> 2.0"]}, - {:phoenix_live_reload, "1.0.0", [phoenix: "~> 0.16 or ~> 1.0"]}, - {:phoenix_live_reload, "1.0.3", [phoenix: "~> 0.16 or ~> 1.0"]}, - {:phoenix_ecto, "2.0.0", [ecto: "~> 1.1", poison: "~> 1.3"]}, - {:phoenix_ecto, "2.0.1", [ecto: "~> 1.1", poison: "~> 1.3"]}, - {:ecto, "1.1.0", [poison: "~> 1.0"]}, - - {:beta, "1.0.0", []}, - {:beta, "1.1.0-beta", []}] + [{:bar, "0.0.1", []}, + {:bar, "0.1.0", [foo: "~> 0.1.0"]}, + {:bar, "0.2.0", [foo: "~> 0.2.0"]}, + {:beta, "1.0.0", []}, + {:beta, "1.1.0-beta", []}, + {:decimal, "0.0.1", []}, + {:decimal, "0.1.0", []}, + {:decimal, "0.2.0", []}, + {:decimal, "0.2.1", []}, + {:depend_name, "0.2.0", [{:package_name, ">= 0.0.0", false, :app_name}]}, + {:ecto, "0.2.0", [postgrex: "~> 0.2.0", ex_doc: "~> 0.0.1"]}, + {:ecto, "0.2.1", [postgrex: "~> 0.2.1", ex_doc: "0.1.0"]}, + {:ecto, "1.1.0", [poison: "~> 1.0"]}, + {:eric, "0.0.1", []}, + {:eric, "0.0.2", []}, + {:eric, "0.1.0", [jose: "~> 0.1.0"]}, + {:eric, "0.1.2", [jose: "~> 0.1.0"]}, + {:eric, "0.2.0", [jose: "~> 0.3.0"]}, + {:ex_doc, "0.0.1", []}, + {:ex_doc, "0.0.2", []}, + {:ex_doc, "0.1.0", []}, + {:ex_plex, "0.0.1", []}, + {:ex_plex, "0.0.2", [decimal: "0.1.1"]}, + {:ex_plex, "0.1.0", [decimal: "~> 0.1.0"]}, + {:ex_plex, "0.1.2", [decimal: "~> 0.1.0"]}, + {:ex_plex, "0.2.0", [decimal: "~> 0.2.0"]}, + {:foo, "0.0.1", []}, + {:foo, "0.1.0", []}, + {:foo, "0.2.0", []}, + {:foo, "0.2.1", []}, + {:has_optional, "0.1.0", [{:ex_doc, "~> 0.0.1", true}]}, + {:jose, "0.2.0", []}, + {:jose, "0.2.1", []}, + {:only_doc, "0.1.0", [{:ex_doc, ">= 0.0.0", true}]}, + {:package_name, "0.1.0", []}, + {:phoenix, "0.0.1", [postgrex: "~> 0.2"]}, + {:phoenix, "1.1.2", [poison: "~> 1.5 or ~> 2.0"]}, + {:phoenix, "1.1.3", [poison: "~> 1.5 or ~> 2.0"]}, + {:poison, "1.5.2", []}, + {:poison, "2.0.0", []}, + {:phoenix_ecto, "2.0.0", [ecto: "~> 1.1", poison: "~> 1.3"]}, + {:phoenix_ecto, "2.0.1", [ecto: "~> 1.1", poison: "~> 1.3"]}, + {:phoenix_live_reload, "1.0.0", [phoenix: "~> 0.16 or ~> 1.0"]}, + {:phoenix_live_reload, "1.0.3", [phoenix: "~> 0.16 or ~> 1.0"]}, + {:postgrex, "0.2.0", [ex_doc: "0.0.1"]}, + {:postgrex, "0.2.1", [ex_doc: "~> 0.1.0"]},] end def setup_auth(username, password) do @@ -182,39 +177,46 @@ defmodule HexTest.Case do {:ok, _} = Hex.State.start_link Hex.State.put(:home, Path.expand("../../tmp/hex_home", __DIR__)) - Hex.State.put(:registry_updated, false) Hex.State.put(:hexpm_pk, File.read!(Path.join(__DIR__, "../fixtures/test_pub.pem"))) Hex.State.put(:api, "http://localhost:4043/api") Hex.State.put(:mirror, System.get_env("HEX_MIRROR") || "http://localhost:4043/repo") Hex.State.put(:pbkdf2_iters, 10) Hex.State.put(:clean_pass, false) - @hex_state Hex.State.get_all - Hex.State.stop def reset_state do Hex.State.put_all(@hex_state) end - setup_all do - ets_path = tmp_path("registry.ets") - File.rm(ets_path) - create_test_registry(ets_path) + setup_all context do + unless context[:async] do + ets_path = tmp_path("cache.ets") + File.rm(ets_path) + create_test_registry(ets_path) + end :ok end - setup do - reset_state() - - Hex.Parallel.clear(:hex_fetcher) - Hex.Registry.ETS.close + setup context do + unless context[:async] do + {:ok, pid} = Hex.Registry.Server.start_link + on_exit(fn -> + ref = Process.monitor(pid) + receive do + {:DOWN, ^ref, :process, ^pid, _info} -> + :ok + end + end) - Mix.shell(Mix.Shell.Process) - Mix.Task.clear - Mix.Shell.Process.flush - Mix.ProjectStack.clear_cache - Mix.ProjectStack.clear_stack + reset_state() + Hex.Parallel.clear(:hex_fetcher) + Mix.shell(Mix.Shell.Process) + Mix.Task.clear + Mix.Shell.Process.flush + Mix.ProjectStack.clear_cache + Mix.ProjectStack.clear_stack + end :ok end @@ -227,7 +229,7 @@ defmodule HexTest.Case do case conn do %Plug.Conn{request_path: "/docs/package-1.1.2.tar.gz"} -> tar_file = tmp_path("package-1.1.2.tar.gz") - index_file = String.to_char_list("index.html") + index_file = Hex.string_to_charlist("index.html") :erl_tar.create(tar_file, [{index_file, ""}], [:compressed]) package = File.read!(tar_file) Plug.Conn.resp(conn, 200, package) diff --git a/test/support/hex_web.ex b/test/support/hex_web.ex index 317b6adc7..f3a09c9a8 100644 --- a/test/support/hex_web.ex +++ b/test/support/hex_web.ex @@ -54,13 +54,13 @@ defmodule HexTest.HexWeb do end def start do - path = String.to_char_list(path()) - hexweb_mix_home = String.to_char_list(hexweb_mix_home()) - hexweb_mix_archives = String.to_char_list(hexweb_mix_archives()) + path = Hex.string_to_charlist(path()) + hexweb_mix_home = Hex.string_to_charlist(hexweb_mix_home()) + hexweb_mix_archives = Hex.string_to_charlist(hexweb_mix_archives()) key = Path.join(__DIR__, "../fixtures/test_priv.pem") |> File.read! - |> String.to_char_list + |> Hex.string_to_charlist env = [ {'MIX_ENV', 'hex'}, @@ -103,7 +103,7 @@ defmodule HexTest.HexWeb do unless File.exists?(dir) do IO.puts "Unable to find #{dir}, make sure to clone the hex_web repository " <> - "into it to run integration tests" + "into it to run integration tests or set HEXWEB_PATH to its location" System.halt(1) end end @@ -114,7 +114,7 @@ defmodule HexTest.HexWeb do defp hexweb_mix do if path = hexweb_elixir() do - path = String.to_char_list(path) + path = Hex.string_to_charlist(path) :os.find_executable('mix', path) else :os.find_executable('mix') @@ -123,13 +123,17 @@ defmodule HexTest.HexWeb do defp hexweb_elixir do if path = System.get_env("HEXWEB_ELIXIR_PATH") do - path |> Path.expand |> Path.join("bin") + path + |> Path.expand + |> Path.join("bin") end end defp hexweb_otp do if path = System.get_env("HEXWEB_OTP_PATH") do - path |> Path.expand |> Path.join("bin") + path + |> Path.expand + |> Path.join("bin") end end @@ -207,8 +211,9 @@ defmodule HexTest.HexWeb do {app, req} -> {app, %{app: app, requirement: req, optional: false}} {app, req, opts} -> + opts = Enum.into(opts, %{}) default_opts = %{app: app, requirement: req, optional: false} - {opts[:hex] || app, Dict.merge(default_opts, opts)} + {opts[:hex] || app, Map.merge(default_opts, opts)} end) meta = diff --git a/test/support/release_samples.ex b/test/support/release_samples.ex index 601ca3df5..06cf9ff2d 100644 --- a/test/support/release_samples.ex +++ b/test/support/release_samples.ex @@ -5,7 +5,7 @@ defmodule ReleaseSimple.Mixfile do version: "0.0.1", package: [licenses: ["MIT"], maintainers: ["maintainers"], - links: %{"a" => "b"}]] + links: %{"a" => "http://a"}]] end end @@ -23,7 +23,7 @@ defmodule ReleaseMeta.Mixfile do description: "foo", package: [files: ["myfile.txt", "missing.txt", "missing/*"], licenses: ["Apache"], - links: %{"a" => "b"}, + links: %{"a" => "http://a"}, maintainers: ["maintainers"], extra: %{"c" => "d"}]] end @@ -35,7 +35,7 @@ defmodule ReleaseName.Mixfile do package: [name: :released_name, licenses: ["MIT"], maintainers: ["maintainers"], - links: %{"a" => "b"}]] + links: %{"a" => "http://a"}]] end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 615eb8dad..afcde8bb7 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -19,7 +19,7 @@ unless :integration in ExUnit.configuration[:exclude] do "maintainers" => ["John Doe", "Jane Doe"], "licenses" => ["GPL2", "MIT", "Apache"], "links" => %{"docs" => "http://docs", "repo" => "http://repo"}, - "description" => "Builds docs" + "description" => "Some description" } auth = HexWeb.new_user("user", "user@mail.com", "hunter42", "my_key") @@ -39,6 +39,7 @@ unless :integration in ExUnit.configuration[:exclude] do HexWeb.new_package("foo", "0.1.0", [], pkg_meta, auth) HexWeb.new_package("foo", "0.1.1", [], pkg_meta, auth) HexWeb.new_package("bar", "0.1.0", [foo: "~> 0.1.0"], pkg_meta, auth) + HexWeb.new_package("baz", "0.1.0", [foo: "0.1.0"], pkg_meta, auth) HexWeb.new_package("beta", "1.0.0", [], pkg_meta, auth) HexWeb.new_package("beta", "1.1.0-beta", [], pkg_meta, auth) end