diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b342e402a..6dbbd4f2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,7 +126,7 @@ jobs: POSTGRES_USER: docker POSTGRES_PASSWORD: d0ck3r options: >- - --health-cmd "pg_isready -U postgres" + --health-cmd "pg_isready -U docker" --health-interval 10s --health-timeout 5s --health-retries 5 @@ -221,6 +221,8 @@ jobs: restore-keys: | ${{ runner.os }}-npm-v6-${{ hashFiles('**/package-lock.json') }} ${{ runner.os }}-npm-v6- + - name: Set SECRET_KEY_BASE + run: echo "SECRET_KEY_BASE=$(openssl rand -hex 32)" >> $GITHUB_ENV - name: Elixir Static Analysis run: mix credo working-directory: app diff --git a/.gitignore b/.gitignore index 73f62afa2..473630e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ npm-debug.log *.tfvars /infrastructure/**/.terraform/ +/infrastructure/**/*.tfstate* /infrastructure/**/_build/ /infrastructure/**/build/ /infrastructure/**/builds/ diff --git a/.tool-versions b/.tool-versions index c57ba80b8..72420f11d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,6 +1,6 @@ nodejs 18.18.0 erlang 26.0.2 elixir 1.15.4-otp-26 -terraform 1.1.5 +terraform 1.7.4 tflint 0.34.1 #npm 10.1.0 diff --git a/app/config/.credo.exs b/app/config/.credo.exs index 0dd21bc21..462f44e28 100644 --- a/app/config/.credo.exs +++ b/app/config/.credo.exs @@ -7,6 +7,7 @@ excluded: [] }, checks: [ + {Credo.Check.Design.TagTODO, false}, {Credo.Check.Refactor.MapInto, false}, {Credo.Check.Warning.LazyLogging, false} ] diff --git a/app/config/config.exs b/app/config/config.exs index 58a191c30..41c7a2b59 100644 --- a/app/config/config.exs +++ b/app/config/config.exs @@ -1,207 +1,44 @@ -# This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. +# Meadow configuration takes place across multiple files. # -# This configuration file is loaded before any dependency and -# is restricted to this project. +# Compile-time configs are evaluated when Meadow is compiled, with +# higher priority given to files farther down the list (i.e., environment- +# specific configs take precedence over global config). +# +# config/config.exs +# config/[environment].exs +# +# Runtime configs work the same way, except they are set at runtime. +# This means they can read configuration data that may not be available +# at compile-time, or that may change. This includes files on disk, +# the runtime system environment, or remote datastores such as AWS +# Secrets Manager. +# +# lib/meadow/config/runtime.ex +# lib/meadow/config/runtime/[environment].ex +# config/[environment].local.exs +# +# When one configuration takes precedence over other, keyword lists are +# deep merged, while other values (e.g., maps) are overwritten. +# +# It is _strongly_ suggested that component configurations be done entirely +# at compile-time _or_ runtime and not some of each, e.g.: +# don't try to +# ``` +# config :meadow, :component, [static properties] +# ``` +# at compile time and +# ``` +# config :meadow, :component, [dynamic properties] +# ``` +# at runtime. # General application configuration import Config -import Env -alias Meadow.Utils.Hush.Transformer, as: CustomTransformer +alias Meadow.Pipeline.Actions config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase -config :meadow, - ecto_repos: [Meadow.Repo], - environment: Mix.env(), - environment_prefix: prefix() - -config :meadow, Meadow.Repo, - migration_primary_key: [ - name: :id, - type: :binary_id, - autogenerate: false, - read_after_writes: true, - default: {:fragment, "uuid_generate_v4()"} - ], - migration_timestamps: [type: :utc_datetime_usec], - username: aws_secret("meadow", dig: ["db", "user"], default: "docker"), - password: aws_secret("meadow", dig: ["db", "password"], default: "d0ck3r"), - database: aws_secret("meadow", dig: ["db", "database"], default: prefix("meadow")), - hostname: aws_secret("meadow", dig: ["db", "host"], default: "localhost"), - port: aws_secret("meadow", dig: ["db", "port"], default: 5432) - -# Configures the endpoint -config :meadow, MeadowWeb.Endpoint, - url: [host: "localhost"], - secret_key_base: "C7BC/yBsTCe/PaJ9g0krwlQrNZZV2r3jSjeuGCeIu9mfNE+4bPcNPHiINQtIQk/B", - render_errors: [view: MeadowWeb.ErrorView, accepts: ~w(html json)], - pubsub_server: Meadow.PubSub, - live_view: [signing_salt: "C7BC/yBsTCe/PaJ9g0krwlQrNZZV2r3jSjeuGCeIu9mfNE+4bPcNPHiINQtIQk/B"] - -# Configures the Search cluster -config :meadow, Meadow.Search.Cluster, - url: - aws_secret("meadow", - dig: ["search", "cluster_endpoint"], - default: "http://localhost:9201" - ), - indexes: [ - %{ - name: prefix("dc-v2-work"), - settings: "priv/search/v2/settings/work.json", - version: 2, - schemas: [Meadow.Data.Schemas.Work], - pipeline: prefix("dc-v2-work-pipeline") - }, - %{ - name: prefix("dc-v2-file-set"), - settings: "priv/search/v2/settings/file_set.json", - version: 2, - schemas: [Meadow.Data.Schemas.FileSet] - }, - %{ - name: prefix("dc-v2-collection"), - settings: "priv/search/v2/settings/collection.json", - version: 2, - schemas: [Meadow.Data.Schemas.Collection] - } - ], - default_options: [ - timeout: 20_000, - recv_timeout: 30_000 - ], - bulk_page_size: 200, - bulk_wait_interval: 500, - json_encoder: Ecto.Jason, - embedding_model_id: - aws_secret("meadow", - dig: ["search", "embedding_model_id"], - default: nil - ), - embedding_dimensions: - aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil), - embedding_text_fields: [ - :title, - :alternate_title, - :description, - :collection, - :creator, - :contributor, - :date_created, - :genre, - :subject, - :style_period, - :language, - :location, - :publisher, - :technique, - :physical_description_material, - :physical_description_size, - :caption, - :table_of_contents, - :scope_and_contents, - :abstract, - ] - -config :meadow, - ark: %{ - default_shoulder: aws_secret("meadow", dig: ["ezid", "shoulder"], default: "ark:/12345/nu1"), - user: aws_secret("meadow", dig: ["ezid", "user"], default: "ark_user"), - password: aws_secret("meadow", dig: ["ezid", "password"], default: "ark_password"), - target_url: - aws_secret("meadow", - dig: ["ezid", "target_base_url"], - default: "https://devbox.library.northwestern.edu:3333/items/" - ), - url: aws_secret("meadow", dig: ["ezid", "url"], default: "http://localhost:3943") - }, - ingest_bucket: prefix("ingest"), - upload_bucket: prefix("uploads"), - pipeline_delay: :timer.seconds(5), - preservation_bucket: prefix("preservation"), - preservation_check_bucket: prefix("preservation-checks"), - pyramid_bucket: prefix("pyramids"), - streaming_bucket: prefix("streaming"), - streaming_url: - aws_secret("meadow", - dig: ["streaming", "base_url"], - default: "https://#{prefix()}-streaming.s3.amazonaws.com/" - ), - mediaconvert_client: MediaConvert, - multipart_upload_concurrency: System.get_env("MULTIPART_UPLOAD_CONCURRENCY", "10"), - iiif_server_url: - aws_secret("meadow", - dig: ["iiif", "base_url"], - default: "https://iiif.dev.rdc.library.northwestern.edu/iiif/3/#{prefix()}" - ), - iiif_manifest_url_deprecated: - aws_secret("meadow", - dig: ["iiif", "manifest_url"], - default: "https://#{prefix()}-pyramids.s3.amazonaws.com/public/" - ), - iiif_distribution_id: aws_secret("meadow", dig: ["iiif", "distribution_id"], default: nil), - digital_collections_url: - aws_secret("meadow", - dig: ["dc", "base_url"], - default: "https://dc.rdc-staging.library.northwestern.edu/" - ), - progress_ping_interval: System.get_env("PROGRESS_PING_INTERVAL", "1000"), - validation_ping_interval: System.get_env("VALIDATION_PING_INTERVAL", "1000"), - shared_links_index: prefix("shared_links"), - pyramid_tiff_working_dir: System.tmp_dir!(), - streaming_distribution_id: - aws_secret("meadow", dig: ["streaming", "distribution_id"], default: nil), - work_archiver_endpoint: aws_secret("meadow", dig: ["work_archiver", "endpoint"], default: "") - -config :exldap, :settings, - server: aws_secret("meadow", dig: ["ldap", "host"], default: "localhost"), - base: - aws_secret("meadow", - dig: ["ldap", "base"], - default: "DC=library,DC=northwestern,DC=edu" - ), - port: aws_secret("meadow", dig: ["ldap", "port"], cast: :integer, default: 390), - user_dn: - aws_secret("meadow", - dig: ["ldap", "user_dn"], - default: "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu" - ), - password: aws_secret("meadow", dig: ["ldap", "password"], default: "d0ck3rAdm1n!"), - ssl: aws_secret("meadow", dig: ["ldap", "ssl"], cast: :boolean, default: false) - -config :meadow, - transcoding_presets: %{ - audio: [ - %{NameModifier: "-high", Preset: "meadow-audio-high"}, - %{NameModifier: "-medium", Preset: "meadow-audio-medium"} - ], - video: [ - %{NameModifier: "-1080", Preset: "meadow-video-high"}, - %{NameModifier: "-720", Preset: "meadow-video-medium"}, - %{NameModifier: "-540", Preset: "meadow-video-low"} - ] - } - -# Configure checksum requirements -config :meadow, - required_checksum_tags: ["computed-md5"], - checksum_wait_timeout: 3_600_000 - -# Configure Lambda-based actions -lambda_from_ssm = fn lambda, function -> - {:lambda, aws_secret("meadow", dig: ["pipeline", lambda], default: "#{function}:$LATEST")} -end - -config :meadow, :lambda, - digester: lambda_from_ssm.("digester", "digester"), - exif: lambda_from_ssm.("exif", "exif"), - frame_extractor: lambda_from_ssm.("frame_extractor", "frame-extractor"), - mediainfo: lambda_from_ssm.("mediainfo", "mediainfo"), - mime_type: lambda_from_ssm.("mime_type", "mime-type"), - tiff: lambda_from_ssm.("tiff", "pyramid-tiff") - # Configures Elixir's Logger config :logger, :console, format: "$time $metadata[$level] $message\n", @@ -210,8 +47,6 @@ config :logger, :console, # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason -config :ueberauth, Ueberauth, providers: [nusso: {Ueberauth.Strategy.NuSSO, []}] - config :authoritex, authorities: [ Authoritex.FAST.CorporateName, @@ -235,20 +70,6 @@ config :authoritex, NUL.Authority ] -config :honeybadger, - api_key: environment_secret("HONEYBADGER_API_KEY", default: "DO_NOT_REPORT"), - environment_name: System.get_env("HONEYBADGER_ENVIRONMENT", to_string(Mix.env())), - revision: System.get_env("HONEYBADGER_REVISION", ""), - repos: [Meadow.Repo], - breadcrumbs_enabled: true, - filter: Meadow.Error.Filter, - exclude_envs: [:dev, :test] - -config :ex_aws, - access_key_id: [:instance_role], - secret_access_key: [:instance_role], - region: System.get_env("AWS_REGION", "us-east-1") - config :httpoison_retry, wait: 50 config :meadow, :extra_mime_types, %{ @@ -266,28 +87,26 @@ config :meadow, :extra_mime_types, %{ "xml" => "application/xml" } -config :meadow, - dc_api: [ - v2: aws_secret("meadow", dig: ["dc_api", "v2"]) +config :meadow, Meadow.Pipeline, + actions: [ + Actions.IngestFileSet, + Actions.ExtractMimeType, + Actions.InitializeDispatch, + Actions.GenerateFileSetDigests, + Actions.ExtractExifMetadata, + Actions.CopyFileToPreservation, + Actions.CreateDerivativeCopy, + Actions.CreatePyramidTiff, + Actions.ExtractMediaMetadata, + Actions.CreateTranscodeJob, + Actions.TranscodeComplete, + Actions.GeneratePosterImage, + Actions.FileSetComplete ] -config :hush, - transformers_override: true, - transformers: [ - CustomTransformer.Dig, - CustomTransformer.Default, - CustomTransformer.Split, - Hush.Transformer.Apply, - CustomTransformer.Cast, - Hush.Transformer.ToFile +config :ueberauth, Ueberauth, + providers: [ + nusso: {Ueberauth.Strategy.NuSSO, []} ] -import_config "pipeline.exs" - -# Import environment specific config. This must remain at the bottom -# of this file so it overrides the configuration defined above. - import_config("#{Mix.env()}.exs") - -if File.exists?("config/#{Mix.env()}.local.exs"), - do: import_config("#{Mix.env()}.local.exs") diff --git a/app/config/dev.exs b/app/config/dev.exs index bfa8a39f4..5eac5ede8 100644 --- a/app/config/dev.exs +++ b/app/config/dev.exs @@ -1,47 +1,9 @@ import Config -import Env -alias Hush.Provider.AwsSecretsManager - -File.mkdir_p!("priv/cert") - -config :meadow, Meadow.Repo, - show_sensitive_data_on_connection_error: true, - timeout: 60_000, - connect_timeout: 60_000, - handshake_timeout: 60_000, - pool_size: 50 - -config :meadow, MeadowWeb.Endpoint, - https: [ - port: 3001, - cipher_suite: :strong, - certfile: - aws_secret("wildcard_ssl", - apply: &{:ok, Map.get(&1, "certificate")}, - to_file: "priv/cert/cert.pem" - ), - keyfile: - aws_secret("wildcard_ssl", apply: &{:ok, Map.get(&1, "key")}, to_file: "priv/cert/key.pem") - ], - debug_errors: false, - code_reloader: true, - check_origin: false, - watchers: [ - node: [ - "build.js", - "--watch", - cd: Path.expand("../assets", __DIR__) - ] - ], - live_reload: [ - patterns: [ - ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$", - ~r"priv/gettext/.*(po)$", - ~r"lib/meadow_web/{live,views}/.*(ex)$", - ~r"lib/meadow_web/templates/.*(eex)$" - ] - ] +config :ex_aws, + access_key_id: [:instance_role], + secret_access_key: [:instance_role], + region: System.get_env("AWS_REGION", "us-east-1") if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do # Configures lambda scripts @@ -72,19 +34,6 @@ if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do end) end -config :meadow, - index_interval: 30_000, - mediaconvert_queue: prefix("transcode"), - mediaconvert_role: aws_secret("meadow", dig: ["transcode", "role_arn"]) - -config :meadow, Meadow.Scheduler, - overlap: false, - timezone: "America/Chicago", - jobs: [ - # Runs every 10 minutes: - {"*/10 * * * *", {Meadow.Data.PreservationChecks, :start_job, []}} - ] - # Do not include metadata nor timestamps in development logs config :logger, :console, format: "$metadata[$level] $message\n", @@ -96,36 +45,3 @@ config :phoenix, :stacktrace_depth, 20 # Initialize plugs at runtime for faster development compilation config :phoenix, :plug_init_mode, :runtime - -config :ueberauth, Ueberauth, - providers: [ - nusso: - {Ueberauth.Strategy.NuSSO, - [ - base_url: aws_secret("meadow", dig: ["nusso", "base_url"]), - callback_path: "/auth/nusso/callback", - callback_port: 3001, - consumer_key: aws_secret("meadow", dig: ["nusso", "api_key"]), - include_attributes: false, - ssl_port: 3001 - ]} - ] - -if prefix = System.get_env("DEV_PREFIX") do - config :meadow, - dc_api: [ - v2: %{ - "api_token_secret" => - aws_secret("meadow", dig: ["dc_api", "v2", "api_token_secret"], default: "DEV_SECRET"), - "api_token_ttl" => 300, - "base_url" => "https://#{prefix}.dev.rdc.library.northwestern.edu:3002" - } - ] -end - -config :meadow, :sitemaps, - base_url: "https://dc.library.northwestern.edu/", - gzip: false, - store: Sitemapper.FileStore, - sitemap_url: "https://devbox.library.northwestern.edu:3333/", - store_config: [path: "priv/static"] diff --git a/app/config/pipeline.exs b/app/config/pipeline.exs deleted file mode 100644 index 8a2ad853b..000000000 --- a/app/config/pipeline.exs +++ /dev/null @@ -1,127 +0,0 @@ -import Config -import Env - -prefix = - case System.get_env("DEV_PREFIX") do - nil -> "meadow" - _ -> prefix() - end - -alias Meadow.Pipeline.Actions.{ - CopyFileToPreservation, - CreateDerivativeCopy, - CreatePyramidTiff, - CreateTranscodeJob, - ExtractMediaMetadata, - ExtractMimeType, - FileSetComplete, - GenerateFileSetDigests, - GeneratePosterImage, - ExtractExifMetadata, - IngestFileSet, - InitializeDispatch, - TranscodeComplete -} - -config :meadow, Meadow.Pipeline, [ - {:actions, - [ - IngestFileSet, - ExtractMimeType, - InitializeDispatch, - GenerateFileSetDigests, - ExtractExifMetadata, - CopyFileToPreservation, - CreateDerivativeCopy, - CreatePyramidTiff, - ExtractMediaMetadata, - CreateTranscodeJob, - TranscodeComplete, - GeneratePosterImage, - FileSetComplete - ]}, - {IngestFileSet, - producer: [ - queue_name: "#{prefix}-ingest-file-set", - wait_time_seconds: 1 - ], - processors: [default: [concurrency: 10]]}, - {ExtractMimeType, - producer: [ - queue_name: "#{prefix}-extract-mime-type", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {InitializeDispatch, - producer: [ - queue_name: "#{prefix}-initialize-dispatch", - wait_time_seconds: 1 - ]}, - {GenerateFileSetDigests, - producer: [ - queue_name: "#{prefix}-generate-file-set-digests", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {CopyFileToPreservation, - producer: [ - queue_name: "#{prefix}-copy-file-to-preservation", - wait_time_seconds: 1, - visibility_timeout: 300 - ]}, - {ExtractExifMetadata, - producer: [ - queue_name: "#{prefix}-extract-exif-metadata", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {ExtractMediaMetadata, - producer: [ - queue_name: "#{prefix}-extract-media-metadata", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {CreatePyramidTiff, - producer: [ - queue_name: "#{prefix}-create-pyramid-tiff", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {CreateDerivativeCopy, - producer: [ - queue_name: "#{prefix}-create-derivative-copy", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {CreateTranscodeJob, - producer: [ - queue_name: "#{prefix}-create-transcode-job", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {TranscodeComplete, - producer: [ - queue_name: "#{prefix}-transcode-complete", - wait_time_seconds: 1, - visibility_timeout: 300 - ], - processors: [default: [concurrency: 1]]}, - {GeneratePosterImage, - producer: [ - queue_name: "#{prefix}-generate-poster-image", - wait_time_seconds: 1, - visibility_timeout: 300 - ]}, - {FileSetComplete, - producer: [ - queue_name: "#{prefix}-file-set-complete", - wait_time_seconds: 1 - ]} -] diff --git a/app/config/prod.exs b/app/config/prod.exs index 17031c262..faf67362e 100644 --- a/app/config/prod.exs +++ b/app/config/prod.exs @@ -1,70 +1,6 @@ import Config -alias Hush.Provider.SystemEnvironment -### YOU PROBABLY WANT TO USE RELEASES.EXS INSTEAD ### - -# For production, don't forget to configure the url host -# to something meaningful, Phoenix uses this information -# when generating URLs. -# -# Note we also include the path to a cache manifest -# containing the digested version of static files. This -# manifest is generated by the `mix phx.digest` task, -# which you should run after static files are built and -# before starting your production server. -config :meadow, MeadowWeb.Endpoint, - url: [host: {:hush, SystemEnvironment, "MEADOW_HOSTNAME", default: "example.com"}, port: 80], - cache_static_manifest: "priv/static/cache_manifest.json" - -# Do not print debug messages in production -config :logger, - compile_time_purge_matching: [ - [level_lower_than: :info] - ] - -# ## SSL Support -# -# To get SSL working, you will need to add the `https` key -# to the previous section and set your `:url` port to 443: -# -# config :meadow, MeadowWeb.Endpoint, -# ... -# url: [host: "example.com", port: 443], -# https: [ -# :inet6, -# port: 443, -# cipher_suite: :strong, -# keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"), -# certfile: System.get_env("SOME_APP_SSL_CERT_PATH") -# ] -# -# The `cipher_suite` is set to `:strong` to support only the -# latest and more secure SSL ciphers. This means old browsers -# and clients may not be supported. You can set it to -# `:compatible` for wider support. -# -# `:keyfile` and `:certfile` expect an absolute path to the key -# and cert in disk or a relative path inside priv, for example -# "priv/ssl/server.key". For all supported SSL configuration -# options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1 -# -# We also recommend setting `force_ssl` in your endpoint, ensuring -# no data is ever sent via http, always redirecting to https: -# -# config :meadow, MeadowWeb.Endpoint, -# force_ssl: [hsts: true] -# -# Check `Plug.SSL` for all available options in `force_ssl`. - -# ## Using releases (Elixir v1.9+) -# -# If you are doing OTP releases, you need to instruct Phoenix -# to start each relevant endpoint: -# -config :meadow, MeadowWeb.Endpoint, server: true -# -# Then you can assemble a release by calling `mix release`. -# See `mix help release` for more information. - -# Finally import the config/prod.secret.exs which loads secrets -# and configuration from environment variables. +config :ex_aws, + access_key_id: [:instance_role], + secret_access_key: [:instance_role], + region: System.get_env("AWS_REGION", "us-east-1") diff --git a/app/config/releases.exs b/app/config/releases.exs index b1d9d47b4..837421c3b 100644 --- a/app/config/releases.exs +++ b/app/config/releases.exs @@ -4,204 +4,9 @@ # recommended and you have to remember to add this # file to your .gitignore. import Config -import Env -alias Meadow.Pipeline.Actions - -[:hackney, :ex_aws] |> Enum.each(&Application.ensure_all_started/1) - -priv_path = fn path -> - case :code.priv_dir(:meadow) do - {:error, :bad_name} -> Path.join([".", "priv", path]) - priv_dir -> priv_dir |> to_string() |> Path.join(path) - end -end - -config :elastix, - custom_headers: - {Meadow.Utils.AWS, :add_aws_signature, - [ - environment_secret("AWS_REGION"), - aws_secret("meadow", dig: ["search", "access_key_id"]), - aws_secret("meadow", dig: ["search", "secret_access_key"]) - ]} - -config :exldap, :settings, - port: 636, - ssl: true - -config :meadow, Meadow.Repo, - pool_size: environment_secret("DB_POOL_SIZE", cast: :integer, default: 10), - queue_target: environment_secret("DB_QUEUE_TARGET", cast: :integer, default: 50), - queue_interval: environment_secret("DB_QUEUE_INTERVAL", cast: :integer, default: 1000) - -host = environment_secret("MEADOW_HOSTNAME", default: "example.com") -port = environment_secret("PORT", cast: :integer, default: 4000) - -config :meadow, MeadowWeb.Endpoint, - url: [host: host, port: port], - http: [ - :inet6, - port: port, - protocol_options: [ - idle_timeout: :infinity, - max_header_value_length: 8192 - ] - ], - check_origin: environment_secret("ALLOWED_ORIGINS", split: ~r/,\s*/, default: ""), - secret_key_base: environment_secret("SECRET_KEY_BASE"), - live_view: [signing_salt: environment_secret("SECRET_KEY_BASE")] - -config :meadow, Meadow.Search.Cluster, - default_options: [ - aws: [ - service: "es", - region: environment_secret("AWS_REGION"), - access_key: aws_secret("meadow", dig: ["search", "access_key_id"]), - secret: aws_secret("meadow", dig: ["search", "secret_access_key"]) - ], - timeout: 20_000, - recv_timeout: 90_000 - ], - bulk_page_size: 200, - bulk_wait_interval: 500, - json_library: Jason, - indexes: [ - %{ - name: "dc-v2-work", - settings: priv_path.("search/v2/settings/work.json"), - version: 2, - schemas: [Meadow.Data.Schemas.Work], - pipeline: prefix("dc-v2-work-pipeline") - }, - %{ - name: "dc-v2-file-set", - settings: priv_path.("search/v2/settings/file_set.json"), - version: 2, - schemas: [Meadow.Data.Schemas.FileSet] - }, - %{ - name: "dc-v2-collection", - settings: priv_path.("search/v2/settings/collection.json"), - version: 2, - schemas: [Meadow.Data.Schemas.Collection] - } - ] +config :logger, level: :info config :meadow, environment: :prod, - environment_prefix: nil, - mediaconvert_client: MediaConvert, - mediaconvert_queue: aws_secret("meadow", dig: ["mediaconvert", "queue"]), - mediaconvert_role: aws_secret("meadow", dig: ["mediaconvert", "role_arn"]), - multipart_upload_concurrency: environment_secret("MULTIPART_UPLOAD_CONCURRENCY", default: "50"), - pipeline_delay: environment_secret("PIPELINE_DELAY", default: "120000"), - progress_ping_interval: environment_secret("PROGRESS_PING_INTERVAL", default: "1000"), - shared_links_index: "shared_links", - sitemaps: [ - gzip: true, - store: Sitemapper.S3Store, - store_config: [ - bucket: aws_secret("meadow", dig: ["buckets", "sitemap"]), - path: "/" - ], - base_url: aws_secret("meadow", dig: ["dc", "base_url"]), - sitemap_url: - aws_secret("meadow", dig: ["dc", "base_url"], apply: &{:ok, Path.join(&1, "api/sitemap")}) - ], - validation_ping_interval: environment_secret("VALIDATION_PING_INTERVAL", default: "1000") - -config :meadow, - ingest_bucket: aws_secret("meadow", dig: ["buckets", "ingest"]), - preservation_bucket: aws_secret("meadow", dig: ["buckets", "preservation"]), - pyramid_bucket: aws_secret("meadow", dig: ["buckets", "pyramid"]), - upload_bucket: aws_secret("meadow", dig: ["buckets", "upload"]), - preservation_check_bucket: aws_secret("meadow", dig: ["buckets", "preservation_check"]), - streaming_bucket: aws_secret("meadow", dig: ["buckets", "streaming"]) - -config :meadow, :livebook, url: environment_secret("LIVEBOOK_URL", default: nil) - -config :logger, level: :info - -config :meadow, Meadow.Scheduler, - overlap: false, - timezone: "America/Chicago", - jobs: [ - # Sitemap generation runs daily at the configured time (default: 1AM Central) - { - aws_secret("meadow", dig: ["scheduler", "sitemap"], default: "0 1 * * *"), - {Meadow.Utils.Sitemap, :generate, []} - }, - # Preservation check runs daily at the configured time (default: 2AM Central) - { - aws_secret("meadow", dig: ["scheduler", "preservation_check"], default: "0 2 * * *"), - {Meadow.Data.PreservationChecks, :start_job, []} - } - ] - -config :ueberauth, Ueberauth, - providers: [ - nusso: - {Ueberauth.Strategy.NuSSO, - [ - base_url: - aws_secret("meadow", - dig: ["nusso", "base_url"], - default: "https://northwestern-prod.apigee.net/agentless-websso/" - ), - callback_path: "/auth/nusso/callback", - consumer_key: aws_secret("meadow", dig: ["nusso", "api_key"]), - include_attributes: false - ]} - ] - -config :hackney, - max_connections: environment_secret("HACKNEY_MAX_CONNECTIONS", cast: :integer, default: "1000") - -[ - {Actions.IngestFileSet, "INGEST_FILE_SET"}, - {Actions.ExtractMimeType, "EXTRACT_MIME_TYPE"}, - {Actions.InitializeDispatch, "INITIALIZE_DISPATCH"}, - {Actions.Dispatcher, "DISPATCHER"}, - {Actions.GenerateFileSetDigests, "GENERATE_FILE_SET_DIGESTS"}, - {Actions.ExtractExifMetadata, "EXTRACT_EXIF_METADATA"}, - {Actions.CopyFileToPreservation, "COPY_FILE_TO_PRESERVATION"}, - {Actions.CreatePyramidTiff, "CREATE_PYRAMID_TIFF"}, - {Actions.ExtractMediaMetadata, "EXTRACT_MEDIA_METADATA"}, - {Actions.CreateTranscodeJob, "CREATE_TRANSCODE_JOB"}, - {Actions.TranscodeComplete, "TRANSCODE_COMPLETE"}, - {Actions.GeneratePosterImage, "GENERATE_POSTER_IMAGE"}, - {Actions.FileSetComplete, "FILE_SET_COMPLETE"} -] -|> Enum.each(fn {action, key} -> - with receive_interval <- 1000, - wait_time_seconds <- 1, - max_number_of_messages <- 10 do - config :meadow, - Meadow.Pipeline, - [ - {action, - producer: [ - receive_interval: receive_interval, - wait_time_seconds: wait_time_seconds, - max_number_of_messages: max_number_of_messages, - visibility_timeout: - environment_secret("#{key}_VISIBILITY_TIMEOUT", cast: :integer, default: 300) - ], - processors: [ - default: [ - concurrency: - environment_secret("#{key}_PROCESSOR_CONCURRENCY", - cast: :integer, - default: 10 - ), - max_demand: - environment_secret("#{key}_MAX_DEMAND", cast: :integer, default: 10), - min_demand: environment_secret("#{key}_MIN_DEMAND", cast: :integer, default: 5) - ] - ]} - ] - end -end) - -config :authoritex, geonames_username: aws_secret("meadow", dig: ["geonames", "username"]) + environment_prefix: nil diff --git a/app/config/runtime.exs b/app/config/runtime.exs index 62079403e..3bcaa68c6 100644 --- a/app/config/runtime.exs +++ b/app/config/runtime.exs @@ -1,13 +1,2 @@ -# Resolve app configurations using Hush -# If left to its own devices, Hush.resolve!/0 will try to configure all applications, -# including :kernel and :stdlib, which will output a warning. We can circumvent that -# by excluding them from the list of running applications. - -unless Hush.release_mode?() do - [:hackney, :ex_aws] |> Enum.each(&Application.ensure_all_started/1) - - Application.loaded_applications() - |> Enum.reject(fn {app, _, _} -> Enum.member?([:kernel, :stdlib], app) end) - |> Enum.map(fn {app, _, _} -> {app, Application.get_all_env(app)} end) - |> Hush.resolve!() -end +import Config +Meadow.Config.Runtime.configure!() diff --git a/app/config/test.exs b/app/config/test.exs index 263aeb07c..d3f6f5cce 100644 --- a/app/config/test.exs +++ b/app/config/test.exs @@ -1,92 +1,9 @@ import Config -import Env -# We don't run a server during test. If one is required, -# you can enable the server option below. -config :meadow, MeadowWeb.Endpoint, - http: [port: 4002], - server: false - -config :meadow, Meadow.Search.Cluster, - url: - aws_secret("meadow", - dig: ["search", "cluster_endpoint"], - default: "http://localhost:9200" - ), - bulk_page_size: 3, - bulk_wait_interval: 2, - embedding_model_id: nil - -config :meadow, - index_interval: 1234, - mediaconvert_client: MediaConvert.Mock, - streaming_url: "https://test-streaming-url/", - iiif_server_url: "http://localhost:8184/iiif/3/", - iiif_manifest_url_deprecated: "http://test-pyramids.s3.localhost.localstack.cloud:4566/public/", - digital_collections_url: "https://fen.rdc-staging.library.northwestern.edu/" - -# Configures lambda scripts -config :meadow, :lambda, - digester: {:local, {Path.expand("../lambdas/digester/index.js"), "handler"}}, - exif: {:local, {Path.expand("../lambdas/exif/index.js"), "handler"}}, - frame_extractor: {:local, {Path.expand("../lambdas/frame-extractor/index.js"), "handler"}}, - mediainfo: {:local, {Path.expand("../lambdas/mediainfo/index.js"), "handler"}}, - mime_type: {:local, {Path.expand("../lambdas/mime-type/index.js"), "handler"}}, - tiff: {:local, {Path.expand("../lambdas/pyramid-tiff/index.js"), "handler"}} - -config :meadow, - checksum_notification: %{ - arn: "arn:aws:lambda:us-east-1:000000000000:function:digest-tag", - buckets: ["test-ingest", "test-uploads"] - }, - required_checksum_tags: ["computed-md5"], - checksum_wait_timeout: 15_000 - -config :meadow, - ark: %{ - default_shoulder: "ark:/12345/nu2", - user: "mockuser", - password: "mockpassword", - target_url: "https://devbox.library.northwestern.edu:3333/items/", - url: "http://localhost:3944/" - } - -config :meadow, :elasticsearch_retry, - interval: 100, - max_retries: 3 - -config :authoritex, authorities: [Authoritex.Mock, NUL.Authority] - -config :ueberauth, Ueberauth, - providers: [ - nusso: - {Ueberauth.Strategy.NuSSO, - [ - base_url: "https://northwestern-dev.apigee.net/agentless-websso/", - callback_path: "/auth/nusso/callback", - consumer_key: "test-sso-key", - include_attributes: false - ]} - ] - -config :meadow, Meadow.Repo, - show_sensitive_data_on_connection_error: true, - timeout: 60_000, - connect_timeout: 60_000, - handshake_timeout: 60_000, - pool: Ecto.Adapters.SQL.Sandbox, - queue_target: 5000, - pool_size: 50 - -config :meadow, - dc_api: [ - v2: %{ - "api_token_secret" => "TEST_SECRET", - "api_token_ttl" => 300, - "base_url" => "http://dcapi-test.northwestern.edu" - } - ], - iiif_distribution_id: nil +config :ex_aws, + access_key_id: "fake", + secret_access_key: "fake", + region: "us-east-1" if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do [:mediaconvert, :s3, :secretsmanager, :sns, :sqs] @@ -94,33 +11,12 @@ if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do config :ex_aws, service, scheme: "http://", host: "localhost.localstack.cloud", - port: 4566, - access_key_id: "fake", - secret_access_key: "fake", - region: "us-east-1" + port: 4566 end) end -config :exldap, :settings, base: "OU=test,DC=library,DC=northwestern,DC=edu" - # Print only warnings and errors during test config :logger, level: :info config :logger, :console, format: {Meadow.TestLogHandler, :format} -config :ex_unit, - assert_receive_timeout: 500 - -config :honeybadger, - environment_name: :test, - exclude_envs: [:dev, :test], - api_key: "abc123", - origin: "http://localhost:4444" - -config :meadow, :sitemaps, - gzip: true, - store: Sitemapper.S3Store, - base_url: "http://localhost:3333/", - sitemap_url: "http://localhost:3333/", - store_config: [bucket: prefix("uploads"), path: ""] - config :elixir, :ansi_enabled, true diff --git a/app/lib/env.ex b/app/lib/env.ex deleted file mode 100644 index 8bd91fc6b..000000000 --- a/app/lib/env.ex +++ /dev/null @@ -1,34 +0,0 @@ -if !function_exported?(:Env, :prefix, 0) do - defmodule Env do - @moduledoc """ - Configuration helpers for Meadow Dev, Staging, and Prod - """ - alias Hush.Provider.{AwsSecretsManager, SystemEnvironment} - - def prefix do - env = - cond do - System.get_env("RELEASE_NAME") -> nil - function_exported?(Mix, :env, 0) -> Mix.env() - true -> nil - end - - [System.get_env("DEV_PREFIX"), env] |> Enum.reject(&is_nil/1) |> Enum.join("-") - end - - def prefix(val), do: [prefix(), to_string(val)] |> reject_empty() |> Enum.join("-") - def atom_prefix(val), do: prefix(val) |> String.to_atom() - - def aws_secret(name, opts \\ []), - do: hush_secret(AwsSecretsManager, Path.join(secrets_path(), name), opts) - - def environment_secret(name, opts \\ []), do: hush_secret(SystemEnvironment, name, opts) - def meadow_secret(opts \\ []), do: aws_secret("meadow", opts) - - defp hush_secret(provider, name, opts), do: {:hush, provider, name, opts} - - defp secrets_path, do: System.get_env("SECRETS_PATH", "config") - - defp reject_empty(list), do: Enum.reject(list, &(is_nil(&1) or &1 == "")) - end -end diff --git a/app/lib/meadow/config.ex b/app/lib/meadow/config.ex index 30465e022..eb1b2fef6 100644 --- a/app/lib/meadow/config.ex +++ b/app/lib/meadow/config.ex @@ -143,27 +143,27 @@ defmodule Meadow.Config do @doc "Gather configuration variables as environment for spawned process" def aws_environment do - with config <- Application.get_env(:ex_aws, :s3), + with config <- ExAws.Config.new(:s3), working_dir <- Application.get_env(:meadow, :pyramid_tiff_working_dir) do [] - |> build_environment(config[:access_key_id], "AWS_ACCESS_KEY_ID") - |> build_environment(config[:secret_access_key], "AWS_SECRET_ACCESS_KEY") - |> build_environment(config[:region], "AWS_REGION") + |> build_environment(Map.get(config, :access_key_id), "AWS_ACCESS_KEY_ID") + |> build_environment(Map.get(config, :secret_access_key), "AWS_SECRET_ACCESS_KEY") + |> build_environment(Map.get(config, :region), "AWS_REGION") |> build_environment(extract_endpoint(config), "AWS_S3_ENDPOINT") |> build_environment(working_dir, "TMPDIR") end end defp extract_endpoint(config) do - case config[:host] do + case Map.get(config, :host) do nil -> nil val -> - Keyword.get(config, :scheme, "https://") + Map.get(config, :scheme, "https://") |> URI.parse() |> Map.put(:host, val) - |> Map.put(:port, config[:port]) + |> Map.put(:port, Map.get(config, :port)) |> URI.to_string() end end diff --git a/app/lib/meadow/config/pipeline.ex b/app/lib/meadow/config/pipeline.ex new file mode 100644 index 000000000..ec9e3abd1 --- /dev/null +++ b/app/lib/meadow/config/pipeline.ex @@ -0,0 +1,56 @@ +defmodule Meadow.Config.Pipeline do + @moduledoc """ + Configure Meadow Pipelines + """ + + defp get_config_value(action, var, default) do + key = Module.split(action) |> List.last() |> Inflex.underscore() |> String.upcase() + var_name = Enum.join([key, String.upcase(to_string(var))], "_") + + case System.get_env(var_name) do + nil -> default + val -> String.to_integer(val) + end + end + + def configure!(prefix) do + prefix = + case prefix do + nil -> "meadow" + val -> val + end + + config = Application.get_env(:meadow, Meadow.Pipeline) + + config = + config + |> Keyword.get(:actions) + |> Enum.reduce(config, fn action, acc -> + queue_name = + [ + prefix + | Module.split(action) |> List.last() |> Inflex.underscore() |> String.split("_") + ] + |> Enum.join("-") + + Keyword.put(acc, action, + producer: [ + queue_name: queue_name, + receive_interval: get_config_value(action, :receive_interval, 1000), + wait_time_seconds: get_config_value(action, :wait_time_seconds, 1), + max_number_of_messages: get_config_value(action, :max_number_of_messages, 10), + visibility_timeout: get_config_value(action, :visibility_timeout, 300) + ], + processors: [ + default: [ + concurrency: get_config_value(action, :processor_concurrency, 10), + max_demand: get_config_value(action, :max_demand, 10), + min_demand: get_config_value(action, :min_demand, 5) + ] + ] + ) + end) + + Application.put_env(:meadow, Meadow.Pipeline, config) + end +end diff --git a/app/lib/meadow/config/runtime.ex b/app/lib/meadow/config/runtime.ex new file mode 100644 index 000000000..56f0f1003 --- /dev/null +++ b/app/lib/meadow/config/runtime.ex @@ -0,0 +1,294 @@ +defmodule Meadow.Config.Runtime do + @moduledoc """ + Load and apply Meadow's runtime configuration + """ + + alias Meadow.Config.Pipeline + + import Meadow.Config.Secrets + + # TODO: UPDATE ALL get_secret(:meadow, ["dc",...]) to use DC secrets + + def configure! do + case :ets.info(:secret_cache, :name) do + :secret_cache -> :ets.delete_all_objects(:secret_cache) + :undefined -> :ets.new(:secret_cache, [:set, :protected, :named_table]) + end + + import Config + + [:hackney, :ex_aws] |> Enum.each(&Application.ensure_all_started/1) + + dc_base = + get_secret( + :meadow, + ["dc", "base_url"], + "https://dc.rdc-staging.library.northwestern.edu" + ) + + config :authoritex, geonames_username: get_secret(:meadow, ["geonames", "username"]) + + config :elastix, + custom_headers: {Meadow.Utils.AWS, :add_aws_signature, []} + + config :exldap, :settings, + server: get_secret(:ldap, ["host"]), + base: get_secret(:ldap, ["base"]), + port: get_secret(:ldap, ["port"]), + user_dn: get_secret(:ldap, ["user_dn"]), + password: get_secret(:ldap, ["password"]), + ssl: get_secret(:ldap, ["ssl"]) == "true" + + config :hackney, + max_connections: environment_int("HACKNEY_MAX_CONNECTIONS", 1000) + + config :honeybadger, + api_key: get_secret(:honeybadger, ["api_key"], "DO_NOT_REPORT"), + environment_name: get_secret(:honeybadger, ["environment"], to_string(Mix.env())), + revision: System.get_env("HONEYBADGER_REVISION", ""), + repos: [Meadow.Repo], + breadcrumbs_enabled: true, + filter: Meadow.Error.Filter, + exclude_envs: [:dev, :test] + + config :meadow, + ecto_repos: [Meadow.Repo] + + config :meadow, Meadow.Repo, + username: get_secret(:meadow, ["db", "user"]), + password: get_secret(:meadow, ["db", "password"]), + database: get_secret(:meadow, ["db", "database"], prefix("meadow")), + hostname: get_secret(:meadow, ["db", "host"]), + migration_primary_key: [ + name: :id, + type: :binary_id, + autogenerate: false, + read_after_writes: true, + default: {:fragment, "uuid_generate_v4()"} + ], + migration_timestamps: [type: :utc_datetime_usec], + port: get_secret(:meadow, ["db", "port"]), + pool_size: environment_int("DB_POOL_SIZE", 10), + queue_target: environment_int("DB_QUEUE_TARGET", 50), + queue_interval: environment_int("DB_QUEUE_INTERVAL", 1000) + + host = System.get_env("MEADOW_HOSTNAME", "localhost") + port = environment_int("PORT", 4000) + + config :meadow, MeadowWeb.Endpoint, + url: [host: host, port: port], + http: [ + :inet6, + port: port, + protocol_options: [ + idle_timeout: :infinity, + max_header_value_length: 8192 + ] + ], + check_origin: System.get_env("ALLOWED_ORIGINS", "") |> String.split(~r/,\s*/), + secret_key_base: System.get_env("SECRET_KEY_BASE"), + live_view: [signing_salt: System.get_env("SECRET_KEY_BASE")], + render_errors: [view: MeadowWeb.ErrorView, accepts: ~w(html json)], + pubsub_server: Meadow.PubSub + + config :meadow, Meadow.Search.Cluster, + url: get_secret(:index, ["endpoint"]), + default_options: [ + timeout: 20_000, + recv_timeout: 90_000 + ], + bulk_page_size: 200, + bulk_wait_interval: 500, + indexes: [ + %{ + name: prefix("dc-v2-work"), + settings: priv_path("search/v2/settings/work.json"), + version: 2, + schemas: [Meadow.Data.Schemas.Work], + pipeline: prefix("dc-v2-work-pipeline") + }, + %{ + name: prefix("dc-v2-file-set"), + settings: priv_path("search/v2/settings/file_set.json"), + version: 2, + schemas: [Meadow.Data.Schemas.FileSet] + }, + %{ + name: prefix("dc-v2-collection"), + settings: priv_path("search/v2/settings/collection.json"), + version: 2, + schemas: [Meadow.Data.Schemas.Collection] + } + ], + embedding_model_id: get_secret(:index, ["embedding_model"]), + # TODO: MOVE embedding_dimensions TO INDEX SECRET + embedding_dimensions: get_secret(:index, ["embedding_dimensions"]), + embedding_text_fields: [ + :title, + :alternate_title, + :description, + :collection, + :creator, + :contributor, + :date_created, + :genre, + :subject, + :style_period, + :language, + :location, + :publisher, + :technique, + :physical_description_material, + :physical_description_size, + :caption, + :table_of_contents, + :scope_and_contents, + :abstract + ] + + config :meadow, + ark: %{ + default_shoulder: get_secret(:ezid, ["shoulder"], "ark:/12345/nu1"), + user: get_secret(:ezid, ["user"], "ark_user"), + password: get_secret(:ezid, ["password"], "ark_password"), + target_url: + get_secret( + :meadow, + ["ezid", "target_base_url"], + "https://devbox.library.northwestern.edu:3333/items/" + ), + url: get_secret(:ezid, ["url"], "http://localhost:3943") + } + + config :meadow, + # TODO: REPLACE WITH DC API SECRET + dc_api: [v2: get_secret(:meadow, ["dc_api", "v2"])], + digital_collections_url: + get_secret( + :meadow, + ["dc", "base_url"], + "https://dc.rdc-staging.library.northwestern.edu/" + ), + environment: environment(), + environment_prefix: prefix(), + iiif_server_url: + get_secret( + :iiif, + ["v3"] + ), + iiif_manifest_url_deprecated: + Path.join( + get_secret(:iiif, ["base"]), + "public/" + ), + iiif_distribution_id: get_secret(:iiif, ["distribution_id"]), + mediaconvert_client: MediaConvert, + mediaconvert_queue: get_secret(:meadow, ["mediaconvert", "queue"]), + mediaconvert_role: get_secret(:meadow, ["mediaconvert", "role_arn"]), + multipart_upload_concurrency: environment_int("MULTIPART_UPLOAD_CONCURRENCY", 50), + pipeline_delay: environment_int("PIPELINE_DELAY", 120_000), + progress_ping_interval: environment_int("PROGRESS_PING_INTERVAL", 1000), + pyramid_tiff_working_dir: System.tmp_dir!(), + required_checksum_tags: ["computed-md5"], + checksum_wait_timeout: 3_600_000, + shared_links_index: prefix("shared_links"), + sitemaps: [ + gzip: true, + store: Sitemapper.S3Store, + store_config: [ + bucket: get_secret(:meadow, ["buckets", "sitemap"]), + path: "/" + ], + base_url: dc_base, + sitemap_url: Path.join(dc_base, "api/sitemap") + ], + streaming_distribution_id: get_secret(:meadow, ["streaming", "distribution_id"]), + streaming_url: + get_secret( + :meadow, + ["streaming", "base_url"], + "https://#{prefix()}-streaming.s3.amazonaws.com/" + ), + transcoding_presets: %{ + audio: [ + %{NameModifier: "-high", Preset: "meadow-audio-high"}, + %{NameModifier: "-medium", Preset: "meadow-audio-medium"} + ], + video: [ + %{NameModifier: "-1080", Preset: "meadow-video-high"}, + %{NameModifier: "-720", Preset: "meadow-video-medium"}, + %{NameModifier: "-540", Preset: "meadow-video-low"} + ] + }, + validation_ping_interval: environment_int("VALIDATION_PING_INTERVAL", 1000), + # TODO: UPDATE TO READ FROM API'S SECRETS + work_archiver_endpoint: get_secret(:meadow, ["work_archiver", "endpoint"], "") + + config :meadow, + ingest_bucket: get_secret(:meadow, ["buckets", "ingest"], prefix("ingest")), + preservation_bucket: + get_secret(:meadow, ["buckets", "preservation"], prefix("preservation")), + pyramid_bucket: get_secret(:meadow, ["buckets", "pyramid"], prefix("pyramid")), + upload_bucket: get_secret(:meadow, ["buckets", "upload"], prefix("upload")), + preservation_check_bucket: + get_secret(:meadow, ["buckets", "preservation_check"], prefix("preservation-checks")), + streaming_bucket: get_secret(:meadow, ["buckets", "streaming"], prefix("streaming")) + + config :meadow, :lambda, + digester: {:lambda, get_secret(:meadow, ["pipeline", "digester"], "digester:$LATEST")}, + exif: {:lambda, get_secret(:meadow, ["pipeline", "exif"], "exif:$LATEST")}, + frame_extractor: + {:lambda, get_secret(:meadow, ["pipeline", "frame_extractor"], "frame-extractor:$LATEST")}, + mediainfo: {:lambda, get_secret(:meadow, ["pipeline", "mediainfo"], "mediainfo:$LATEST")}, + mime_type: {:lambda, get_secret(:meadow, ["pipeline", "mime_type"], "mime-type:$LATEST")}, + tiff: {:lambda, get_secret(:meadow, ["pipeline", "tiff"], "pyramid-tiff:$LATEST")} + + config :meadow, :livebook, url: System.get_env("LIVEBOOK_URL") + + config :meadow, Meadow.Scheduler, + overlap: false, + timezone: "America/Chicago", + jobs: [ + # Sitemap generation runs daily at the configured time (default: 1AM Central) + { + get_secret(:meadow, ["scheduler", "sitemap"], "0 1 * * *"), + {Meadow.Utils.Sitemap, :generate, []} + }, + # Preservation check runs daily at the configured time (default: 2AM Central) + { + get_secret(:meadow, ["scheduler", "preservation_check"], "0 2 * * *"), + {Meadow.Data.PreservationChecks, :start_job, []} + } + ] + + config :ueberauth, Ueberauth, + providers: [ + nusso: + {Ueberauth.Strategy.NuSSO, + [ + base_url: + get_secret( + :nusso, + ["base_url"], + "https://northwestern-prod.apigee.net/agentless-websso/" + ), + callback_path: "/auth/nusso/callback", + consumer_key: get_secret(:nusso, ["api_key"]), + include_attributes: false + ]} + ] + + Pipeline.configure!(prefix()) + + with mod <- environment() |> to_string() |> String.capitalize() do + Module.concat(__MODULE__, mod).configure!() + end + + if :code.is_loaded(Mix) do + file = Path.join(File.cwd!(), "config/#{Mix.env()}.local.exs") + if File.exists?(file), do: Code.eval_file(file) + end + + :ok + end +end diff --git a/app/lib/meadow/config/runtime/dev.ex b/app/lib/meadow/config/runtime/dev.ex new file mode 100644 index 000000000..7417f58ab --- /dev/null +++ b/app/lib/meadow/config/runtime/dev.ex @@ -0,0 +1,101 @@ +defmodule Meadow.Config.Runtime.Dev do + @moduledoc """ + Load and apply Meadow's runtime configuration for the dev environment + """ + + import Meadow.Config.Secrets + + defp fetch_cert do + cert_path = project_root() |> Path.join("priv/cert") + File.mkdir_p!(cert_path) + Path.join(cert_path, "cert.pem") |> File.write!(get_secret(:wildcard_ssl, ["certificate"])) + Path.join(cert_path, "key.pem") |> File.write!(get_secret(:wildcard_ssl, ["key"])) + end + + def configure! do + import Config + + fetch_cert() + + config :meadow, Meadow.Repo, + show_sensitive_data_on_connection_error: true, + timeout: 60_000, + connect_timeout: 60_000, + handshake_timeout: 60_000, + pool_size: 50 + + config :meadow, MeadowWeb.Endpoint, + https: [ + port: 3001, + cipher_suite: :strong, + certfile: Path.join(project_root(), "priv/cert/cert.pem"), + keyfile: Path.join(project_root(), "priv/cert/key.pem") + ], + debug_errors: false, + code_reloader: true, + check_origin: false, + watchers: [ + node: [ + "build.js", + "--watch", + cd: Path.join(project_root(), "assets") + ] + ], + live_reload: [ + patterns: [ + ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$", + ~r"priv/gettext/.*(po)$", + ~r"lib/meadow_web/{live,views}/.*(ex)$", + ~r"lib/meadow_web/templates/.*(eex)$" + ] + ] + + config :meadow, + index_interval: 30_000, + mediaconvert_queue: prefix("transcode"), + mediaconvert_role: get_secret(:meadow, ["transcode", "role_arn"]) + + config :meadow, + iiif_server_url: Path.join(get_secret(:iiif, ["v3"]), prefix()), + iiif_manifest_url_deprecated: Path.join(get_secret(:iiif, ["base"]), "public/") + + config :meadow, Meadow.Scheduler, + jobs: [ + # Runs every 10 minutes: + {"*/10 * * * *", {Meadow.Data.PreservationChecks, :start_job, []}} + ] + + config :meadow, :sitemaps, + base_url: "https://dc.library.northwestern.edu/", + gzip: false, + store: Sitemapper.FileStore, + sitemap_url: "https://devbox.library.northwestern.edu:3333/", + store_config: [path: "priv/static"] + + config :ueberauth, Ueberauth, + providers: [ + nusso: + {Ueberauth.Strategy.NuSSO, + [ + base_url: get_secret(:meadow, ["nusso", "base_url"]), + callback_path: "/auth/nusso/callback", + callback_port: 3001, + consumer_key: get_secret(:meadow, ["nusso", "api_key"]), + include_attributes: false, + ssl_port: 3001 + ]} + ] + + if prefix = System.get_env("DEV_PREFIX") do + config :meadow, + dc_api: [ + v2: %{ + "api_token_secret" => + get_secret(:meadow, ["dc_api", "v2", "api_token_secret"], "DEV_SECRET"), + "api_token_ttl" => 300, + "base_url" => "https://#{prefix}.dev.rdc.library.northwestern.edu:3002" + } + ] + end + end +end diff --git a/app/lib/meadow/config/runtime/prod.ex b/app/lib/meadow/config/runtime/prod.ex new file mode 100644 index 000000000..1189577ba --- /dev/null +++ b/app/lib/meadow/config/runtime/prod.ex @@ -0,0 +1,19 @@ +defmodule Meadow.Config.Runtime.Prod do + @moduledoc """ + Load and apply Meadow's runtime configuration for the production environment + """ + + def configure! do + import Config + + config :meadow, MeadowWeb.Endpoint, + url: [host: System.get_env("MEADOW_HOSTNAME", "example.com"), port: 80], + cache_static_manifest: "priv/static/cache_manifest.json", + server: true + + config :logger, + compile_time_purge_matching: [ + [level_lower_than: :info] + ] + end +end diff --git a/app/lib/meadow/config/runtime/test.ex b/app/lib/meadow/config/runtime/test.ex new file mode 100644 index 000000000..58d43297c --- /dev/null +++ b/app/lib/meadow/config/runtime/test.ex @@ -0,0 +1,84 @@ +defmodule Meadow.Config.Runtime.Test do + @moduledoc """ + Load and apply Meadow's runtime configuration for the test environment + """ + + import Meadow.Config.Secrets + + def configure! do + import Config + + # We don't run a server during test. If one is required, + # you can enable the server option below. + config :meadow, MeadowWeb.Endpoint, + http: [port: 4002], + server: false + + config :meadow, + index_interval: 1234, + mediaconvert_client: MediaConvert.Mock + + # Configures lambda scripts + config :meadow, :lambda, + digester: {:local, {Path.expand("../lambdas/digester/index.js"), "handler"}}, + exif: {:local, {Path.expand("../lambdas/exif/index.js"), "handler"}}, + frame_extractor: {:local, {Path.expand("../lambdas/frame-extractor/index.js"), "handler"}}, + mediainfo: {:local, {Path.expand("../lambdas/mediainfo/index.js"), "handler"}}, + mime_type: {:local, {Path.expand("../lambdas/mime-type/index.js"), "handler"}}, + tiff: {:local, {Path.expand("../lambdas/pyramid-tiff/index.js"), "handler"}} + + config :meadow, + checksum_notification: %{ + arn: "arn:aws:lambda:us-east-1:000000000000:function:digest-tag", + buckets: ["test-ingest", "test-upload"] + }, + required_checksum_tags: ["computed-md5"], + checksum_wait_timeout: 15_000 + + config :meadow, :elasticsearch_retry, + interval: 100, + max_retries: 3 + + config :authoritex, authorities: [Authoritex.Mock, NUL.Authority] + + config :meadow, Meadow.Repo, + show_sensitive_data_on_connection_error: true, + timeout: 60_000, + connect_timeout: 60_000, + handshake_timeout: 60_000, + pool: Ecto.Adapters.SQL.Sandbox, + queue_target: 5000, + pool_size: 50 + + config :meadow, Meadow.Search.Cluster, + bulk_page_size: 3, + bulk_wait_interval: 2, + embedding_model_id: nil + + config :meadow, + dc_api: [ + v2: %{ + "api_token_secret" => "TEST_SECRET", + "api_token_ttl" => 300, + "base_url" => "http://dcapi-test.northwestern.edu" + } + ], + iiif_distribution_id: nil + + config :meadow, :sitemaps, + gzip: true, + store: Sitemapper.S3Store, + base_url: "http://localhost:3333/", + sitemap_url: "http://localhost:3333/", + store_config: [bucket: prefix("upload"), path: ""] + + config :ex_unit, + assert_receive_timeout: 500 + + config :honeybadger, + environment_name: :test, + exclude_envs: [:dev, :test], + api_key: "abc123", + origin: "http://localhost:4444" + end +end diff --git a/app/lib/meadow/config/secrets.ex b/app/lib/meadow/config/secrets.ex new file mode 100644 index 000000000..9b3798495 --- /dev/null +++ b/app/lib/meadow/config/secrets.ex @@ -0,0 +1,91 @@ +defmodule Meadow.Config.Secrets do + @moduledoc """ + Functions for retrieving and loading configuration from AWS Secrets Manager and + the local runtime environment + """ + + @config_map %{ + ezid: "infrastructure/ezid", + honeybadger: "infrastructure/honeybadger", + iiif: "infrastructure/iiif", + index: "infrastructure/index", + inference: "infrastructure/inference", + ldap: "infrastructure/ldap", + meadow: "config/meadow", + nusso: "infrastructure/nusso", + wildcard_ssl: "config/wildcard_ssl" + } + + def get_secret(config, path, default \\ nil) do + secrets = + case :ets.lookup(:secret_cache, config) |> Keyword.get(config) do + nil -> + loaded = load_config(@config_map[config]) + :ets.insert(:secret_cache, {config, loaded}) + loaded + + value -> + value + end + + case get_in(secrets, path) do + nil -> default + secret -> secret + end + end + + defp load_config(config_path) do + System.get_env("SECRETS_PATH", nil) |> load_config(config_path) + end + + defp load_config(nil, config_path), do: retrieve_config(config_path) + + defp load_config(prefix, config_path), + do: Path.join(Enum.reject([prefix, config_path], &is_nil/1)) |> retrieve_config() + + defp retrieve_config(path) do + case ExAws.SecretsManager.get_secret_value(path) |> ExAws.request() do + {:ok, %{"SecretString" => secret_string}} -> Jason.decode!(secret_string) + {:error, _} -> nil + end + end + + def environment do + if function_exported?(Mix, :env, 0), do: Mix.env(), else: :prod + end + + def prefix do + env = + cond do + System.get_env("RELEASE_NAME") -> nil + function_exported?(Mix, :env, 0) -> Mix.env() + true -> nil + end + + [System.get_env("DEV_PREFIX"), env] |> Enum.reject(&is_nil/1) |> Enum.join("-") + end + + def prefix(val), do: [prefix(), to_string(val)] |> reject_empty() |> Enum.join("-") + # defp atom_prefix(val), do: prefix(val) |> String.to_atom() + defp reject_empty(list), do: Enum.reject(list, &(is_nil(&1) or &1 == "")) + + def environment_int(key, default) do + case System.get_env(key) do + nil -> default + val -> String.to_integer(val) + end + end + + def project_root do + if Code.loaded?(Mix), + do: Path.dirname(Path.dirname(Mix.Project.build_path())), + else: Path.dirname(:code.priv_dir(:meadow)) + end + + def priv_path(path) do + case :code.priv_dir(:meadow) do + {:error, :bad_name} -> Path.join([".", "priv", path]) + priv_dir -> priv_dir |> to_string() |> Path.join(path) + end + end +end diff --git a/app/lib/meadow/utils/aws.ex b/app/lib/meadow/utils/aws.ex index 1f1b9baf4..0ca267589 100644 --- a/app/lib/meadow/utils/aws.ex +++ b/app/lib/meadow/utils/aws.ex @@ -5,11 +5,11 @@ defmodule Meadow.Utils.AWS do Utility functions for AWS requests and object management """ alias Meadow.Config + alias Meadow.Config.Secrets alias Meadow.Error alias Meadow.Utils.AWS.MultipartCopy alias Meadow.Utils.Pairtree - import Env import SweetXml, only: [sigil_x: 2] require Logger @@ -61,8 +61,8 @@ defmodule Meadow.Utils.AWS do |> request() end - def add_aws_signature(request, region, access_key, secret) do - request.headers ++ generate_aws_signature(request, region, access_key, secret) + def add_aws_signature(request) do + request.headers ++ generate_aws_signature(request) end def check_object_tags!(bucket, key, required_tags) do @@ -79,24 +79,41 @@ defmodule Meadow.Utils.AWS do def copy_object(dest_bucket, dest_object, src_bucket, src_object, opts \\ []), do: MultipartCopy.copy_object(dest_bucket, dest_object, src_bucket, src_object, opts) - def invalidate_cache(file_set, invalidation_type), do: invalidate_cache(file_set, invalidation_type, Config.environment()) - def invalidate_cache(file_set, :pyramid, :dev), do: perform_iiif_invalidation("/iiif/3/#{prefix()}/#{file_set.id}/*") - def invalidate_cache(file_set, :pyramid, :test), do: perform_iiif_invalidation("/iiif/3/#{prefix()}/#{file_set.id}/*") - def invalidate_cache(file_set, :pyramid, _), do: perform_iiif_invalidation("/iiif/3/#{file_set.id}/*") - def invalidate_cache(file_set, :poster, :dev), do: perform_iiif_invalidation("/iiif/3/#{prefix()}/posters/#{file_set.id}/*") - def invalidate_cache(file_set, :poster, :test), do: perform_iiif_invalidation("/iiif/3/#{prefix()}/posters/#{file_set.id}/*") - def invalidate_cache(file_set, :poster, _), do: perform_iiif_invalidation("/iiif/3/posters/#{file_set.id}/*") + def invalidate_cache(file_set, invalidation_type), + do: invalidate_cache(file_set, invalidation_type, Config.environment()) + + def invalidate_cache(file_set, :pyramid, :dev), + do: perform_iiif_invalidation("/iiif/3/#{prefix()}/#{file_set.id}/*") + + def invalidate_cache(file_set, :pyramid, :test), + do: perform_iiif_invalidation("/iiif/3/#{prefix()}/#{file_set.id}/*") + + def invalidate_cache(file_set, :pyramid, _), + do: perform_iiif_invalidation("/iiif/3/#{file_set.id}/*") + + def invalidate_cache(file_set, :poster, :dev), + do: perform_iiif_invalidation("/iiif/3/#{prefix()}/posters/#{file_set.id}/*") + + def invalidate_cache(file_set, :poster, :test), + do: perform_iiif_invalidation("/iiif/3/#{prefix()}/posters/#{file_set.id}/*") + + def invalidate_cache(file_set, :poster, _), + do: perform_iiif_invalidation("/iiif/3/posters/#{file_set.id}/*") + def invalidate_cache(_file_set, :streaming, :dev), do: :ok def invalidate_cache(_file_set, :streaming, :test), do: :ok - def invalidate_cache(file_set, :streaming, _), do: perform_streaming_invalidation("/#{Pairtree.generate!(file_set.id)}/*") - defp perform_iiif_invalidation(path), do: perform_invalidation(path, Config.iiif_cloudfront_distribution_id()) - defp perform_streaming_invalidation(path), do: perform_invalidation(path, Config.streaming_cloudfront_distribution_id()) + def invalidate_cache(file_set, :streaming, _), + do: perform_streaming_invalidation("/#{Pairtree.generate!(file_set.id)}/*") + + defp perform_iiif_invalidation(path), + do: perform_invalidation(path, Config.iiif_cloudfront_distribution_id()) + + defp perform_streaming_invalidation(path), + do: perform_invalidation(path, Config.streaming_cloudfront_distribution_id()) defp perform_invalidation(path, nil) do - Logger.info( - "Skipping cache invalidation for: #{path}. No distribution id found." - ) + Logger.info("Skipping cache invalidation for: #{path}. No distribution id found.") :ok end @@ -135,20 +152,28 @@ defmodule Meadow.Utils.AWS do end end - defp generate_aws_signature(request, region, access_key, secret) do + defp generate_aws_signature(request) do now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) %{host: host} = URI.parse(request.url) + config = ExAws.Config.new(:es) + + headers = + case Map.get(config, :security_token) do + nil -> [{"Host", host}] + security_token -> [{"Host", host}, {"X-Amz-Security-Token", security_token}] + end + :aws_signature.sign_v4( - access_key, - secret, - region, + config.access_key_id, + config.secret_access_key, + config.region, "es", {{now.year, now.month, now.day}, {now.hour, now.minute, now.second}}, request.method |> to_string() |> String.upcase(), request.url, - [{"Host", host}], + headers, request.body, [] ) @@ -256,4 +281,6 @@ defmodule Meadow.Utils.AWS do nil end end + + defp prefix, do: Secrets.prefix() end diff --git a/app/lib/meadow/utils/hush/transformer/cast.ex b/app/lib/meadow/utils/hush/transformer/cast.ex deleted file mode 100644 index 8126cfcec..000000000 --- a/app/lib/meadow/utils/hush/transformer/cast.ex +++ /dev/null @@ -1,46 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.Cast do - @moduledoc """ - Type casting Hush transformer that is more flexible than Hush's built-in Cast - """ - - @behaviour Hush.Transformer - - @impl true - @spec key() :: :cast - def key, do: :cast - - @impl true - @spec transform(config :: any(), value :: any()) :: {:ok, any()} | {:error, String.t()} - def transform(type, value) do - cast!(type, value) - end - - defp cast!(_, nil), do: {:ok, nil} - defp cast!(:binary, value), do: {:ok, to_string(value)} - - defp cast!(type, value) do - {:ok, cast!(type_of(value), type, value)} - rescue - err in ArgumentError -> {:error, err} - end - - defp cast!(from, from, value), do: value - defp cast!(:binary, :atom, value), do: value |> String.to_existing_atom() - defp cast!(:binary, :boolean, value), do: value == "true" - defp cast!(:binary, :charlist, value), do: value |> String.to_charlist() - defp cast!(:binary, :float, value), do: value |> String.to_float() - defp cast!(:binary, :integer, value), do: value |> String.to_integer() - defp cast!(:integer, :float, value), do: value * 1.0 - - defp cast!(_, to, value), do: cast!(:binary, to, to_string(value)) - - defp type_of(value) when is_atom(value), do: :atom - defp type_of(value) when is_binary(value), do: :binary - defp type_of(value) when is_boolean(value), do: :boolean - defp type_of(value) when is_float(value), do: :float - defp type_of(value) when is_integer(value), do: :integer - - defp type_of(value) when is_list(value) do - if Enum.all?(value, &(is_integer(&1) && (0 <= &1 and &1 <= 255))), do: :charlist, else: :list - end -end diff --git a/app/lib/meadow/utils/hush/transformer/default.ex b/app/lib/meadow/utils/hush/transformer/default.ex deleted file mode 100644 index 78186c263..000000000 --- a/app/lib/meadow/utils/hush/transformer/default.ex +++ /dev/null @@ -1,16 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.Default do - @moduledoc """ - Complement Hush's built-in default by operating within the transformer chain - """ - - @behaviour Hush.Transformer - - @impl true - @spec key() :: :default - def key, do: :default - - @impl true - @spec transform(config :: any(), value :: any()) :: {:ok, any()} | {:error, String.t()} - def transform(default, {:error, :not_found}), do: {:ok, default} - def transform(_, value), do: {:ok, value} -end diff --git a/app/lib/meadow/utils/hush/transformer/dig.ex b/app/lib/meadow/utils/hush/transformer/dig.ex deleted file mode 100644 index 515d40e24..000000000 --- a/app/lib/meadow/utils/hush/transformer/dig.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.Dig do - @moduledoc """ - Dig values out of maps as per `Kernel.get_in/2` - """ - @behaviour Hush.Transformer - - @impl true - @spec key() :: :dig - def key, do: :dig - - @impl true - @spec transform(config :: any(), value :: any()) :: {:ok, any()} | {:error, String.t()} - def transform(path, value), do: value |> dig(path) - - defp dig({:ok, {:error, :not_found}} = result, _), do: result - defp dig(value, []), do: {:ok, value} - - defp dig(value, [key | path]) do - Access.get(value, key, {:ok, {:error, :not_found}}) |> dig(path) - rescue - _ in FunctionClauseError -> {:ok, {:error, :not_found}} - end -end diff --git a/app/lib/meadow/utils/hush/transformer/split.ex b/app/lib/meadow/utils/hush/transformer/split.ex deleted file mode 100644 index 9e534d77f..000000000 --- a/app/lib/meadow/utils/hush/transformer/split.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.Split do - @moduledoc """ - Split value on a string or regex - """ - @behaviour Hush.Transformer - - @impl true - @spec key() :: :split - def key, do: :split - - @impl true - @spec transform(config :: any(), value :: any()) :: {:ok, any()} | {:error, String.t()} - def transform({boundary, opts}, value) when is_binary(value), - do: {:ok, value |> String.split(boundary, opts)} - - def transform(boundary, value) when is_binary(value), do: {:ok, value |> String.split(boundary)} - def transform(_, value), do: {:ok, value} -end diff --git a/app/mix.exs b/app/mix.exs index 2ad799d82..344164592 100644 --- a/app/mix.exs +++ b/app/mix.exs @@ -1,5 +1,3 @@ -Code.require_file("lib/env.ex") - defmodule Meadow.MixProject do use Mix.Project @@ -71,6 +69,7 @@ defmodule Meadow.MixProject do {:ex_aws, "~> 2.5.0"}, {:ex_aws_s3, "~> 2.3"}, {:ex_aws_lambda, "~> 2.0"}, + {:ex_aws_secretsmanager, "~> 2.0"}, {:ex_aws_ssm, "~> 2.1"}, {:ex_aws_sts, "~> 2.3.0"}, {:excoveralls, "~> 0.10", only: :test}, @@ -79,8 +78,6 @@ defmodule Meadow.MixProject do {:gettext, "~> 0.11"}, {:hackney, "~> 1.17"}, {:honeybadger, "~> 0.7"}, - {:hush, "~> 0.5"}, - {:hush_aws_secrets_manager, "~> 0.1"}, {:inflex, "~> 2.1.0"}, {:jason, "~> 1.0"}, {:jwt, "~> 0.1.11"}, @@ -140,8 +137,7 @@ defmodule Meadow.MixProject do runtime_tools: :permanent ], extra_applications: [:os_mon], - steps: [&build_assets/1, :assemble], - config_providers: [{Hush.ConfigProvider, nil}] + steps: [&build_assets/1, :assemble] ] ] end diff --git a/app/mix.lock b/app/mix.lock index f456163bd..aa7c188f6 100644 --- a/app/mix.lock +++ b/app/mix.lock @@ -13,15 +13,15 @@ "broadway_sqs": {:hex, :broadway_sqs, "0.7.4", "ab89b298f9253adb8534f92095b56d4879e35fe2f5a0730256f7e824572c637f", [:mix], [{:broadway, "~> 1.0", [hex: :broadway, repo: "hexpm", optional: false]}, {:ex_aws_sqs, "~> 3.2.1 or ~> 3.3", [hex: :ex_aws_sqs, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:saxy, "~> 1.1", [hex: :saxy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7140085c4f7c4b27886b3a8f3d0942976f39f195fdbc2f652c5d7b157f93ae28"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "cachex": {:hex, :cachex, "4.0.2", "120f9c27b0a453c7cb3319d9dc6c61c050a480e5299fc1f8bded1e2e334992ab", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:ex_hash_ring, "~> 6.0", [hex: :ex_hash_ring, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "4f4890122bddd979f6c217d5e300d0c0d3eb858a976cbe1f65a94e6322bc5825"}, - "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "castore": {:hex, :castore, "1.0.9", "5cc77474afadf02c7c017823f460a17daa7908e991b0cc917febc90e466a375c", [:mix], [], "hexpm", "5ea956504f1ba6f2b4eb707061d8e17870de2bee95fb59d512872c2ef06925e7"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "configparser_ex": {:hex, :configparser_ex, "4.0.0", "17e2b831cfa33a08c56effc610339b2986f0d82a9caa0ed18880a07658292ab6", [:mix], [], "hexpm", "02e6d1a559361a063cba7b75bc3eb2d6ad7e62730c551cc4703541fd11e65e5b"}, "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "credo": {:hex, :credo, "1.7.10", "6e64fe59be8da5e30a1b96273b247b5cf1cc9e336b5fd66302a64b25749ad44d", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "71fbc9a6b8be21d993deca85bf151df023a3097b01e09a2809d460348561d8cd"}, - "crontab": {:hex, :crontab, "1.1.13", "3bad04f050b9f7f1c237809e42223999c150656a6b2afbbfef597d56df2144c5", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "d67441bec989640e3afb94e123f45a2bc42d76e02988c9613885dc3d01cf7085"}, - "dataloader": {:hex, :dataloader, "1.0.11", "49bbfc7dd8a1990423c51000b869b1fecaab9e3ccd6b29eab51616ae8ad0a2f5", [:mix], [{:ecto, ">= 3.4.3 and < 4.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0 or ~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba0b0ec532ec68e9d033d03553561d693129bd7cbd5c649dc7903f07ffba08fe"}, + "crontab": {:hex, :crontab, "1.1.14", "233fcfdc2c74510cabdbcb800626babef414e7cb13cea11ddf62e10e16e2bf76", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "4e3b9950bc22ae8d0395ffb5f4b127a140005cba95745abf5ff9ee7e8203c6fa"}, + "dataloader": {:hex, :dataloader, "1.0.11", "49bbfc7dd8a1990423c51000b869b1fecaab9e3ccd6b29eab51616ae8ad0a2f5", [:mix], [{:ecto, ">= 3.4.3 and < 4.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba0b0ec532ec68e9d033d03553561d693129bd7cbd5c649dc7903f07ffba08fe"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, @@ -42,17 +42,15 @@ "ex_hash_ring": {:hex, :ex_hash_ring, "6.0.4", "bef9d2d796afbbe25ab5b5a7ed746e06b99c76604f558113c273466d52fa6d6b", [:mix], [], "hexpm", "89adabf31f7d3dfaa36802ce598ce918e9b5b33bae8909ac1a4d052e1e567d18"}, "excoveralls": {:hex, :excoveralls, "0.18.3", "bca47a24d69a3179951f51f1db6d3ed63bca9017f476fe520eb78602d45f7756", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "746f404fcd09d5029f1b211739afb8fb8575d775b21f6a3908e7ce3e640724c6"}, "exldap": {:hex, :exldap, "0.6.3", "98e8c4ad5e1773b921ba0a5f27c96a6d814a7dff1581e5648eaf7c4f594729e7", [:mix], [], "hexpm", "412fafabe43cfde52d85957776f54edad9258d88b261dab76e0458a43c618a2c"}, - "expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"}, + "expo": {:hex, :expo, "1.1.0", "f7b9ed7fb5745ebe1eeedf3d6f29226c5dd52897ac67c0f8af62a07e661e5c75", [:mix], [], "hexpm", "fbadf93f4700fb44c331362177bdca9eeb8097e8b0ef525c9cc501cb9917c960"}, "faker": {:hex, :faker, "0.18.0", "943e479319a22ea4e8e39e8e076b81c02827d9302f3d32726c5bf82f430e6e14", [:mix], [], "hexpm", "bfbdd83958d78e2788e99ec9317c4816e651ad05e24cfd1196ce5db5b3e81797"}, "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.26.1", "38e14ea5dcf962d1fc9f361b63ea07c0ce715a8ef1f9e82d3dfb8e67e0416715", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "01ce56f188b9dc28780a52783d6529ad2bc7124f9744e571e1ee4ea88bf08734"}, - "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~>2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, + "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, "honeybadger": {:hex, :honeybadger, "0.22.0", "c1476432a5fc0ae6de833f4549065ec7eaf1b9e5fb082d7a58cd312f9d1f9cb1", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:hackney, "~> 1.1", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.0.0 and < 2.0.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, ">= 1.0.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "230ffbd1dc1ad4d38b5dd517e50983e2e5e9b50dc21fc4d3bdbbb6acf90cef50"}, "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "httpoison_retry": {:hex, :httpoison_retry, "1.1.0", "2f2cf49ecac6d1a73d0730b76673890dfd3df35123e83d521e0af7dff6db0cfb", [:mix], [{:httpoison, "~> 0.13 or ~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "df8f33336e2aeb25efd0d8948b51fa2551a2944dc12c8fb739e75a7796923a78"}, - "hush": {:hex, :hush, "0.5.0", "6c95b83dcdb6ea729ce168ea8418e6002cc8a1bda92a1c02d475d7f6303d2c41", [:mix], [], "hexpm", "bc15e3a23a946771b6efbfecda5bc66d466358e4901020997801bf28af5f9e4a"}, - "hush_aws_secrets_manager": {:hex, :hush_aws_secrets_manager, "0.2.0", "308289205282b31168691180f5a85b8ef72d6ae2bac4e3cd5cb6f725a0499b99", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:ex_aws_secretsmanager, "~> 2.0", [hex: :ex_aws_secretsmanager, repo: "hexpm", optional: false]}, {:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}, {:hush, "~> 0.4", [hex: :hush, repo: "hexpm", optional: false]}], "hexpm", "e07d5515d0e9c44ab891c5402ce4f7827cb167eed8ef84b426d8d2962d07a4ab"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, @@ -86,7 +84,7 @@ "quantum": {:hex, :quantum, "3.5.3", "ee38838a07761663468145f489ad93e16a79440bebd7c0f90dc1ec9850776d99", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "500fd3fa77dcd723ed9f766d4a175b684919ff7b6b8cfd9d7d0564d58eba8734"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, - "saxy": {:hex, :saxy, "1.5.0", "0141127f2d042856f135fb2d94e0beecda7a2306f47546dbc6411fc5b07e28bf", [:mix], [], "hexpm", "ea7bb6328fbd1f2aceffa3ec6090bfb18c85aadf0f8e5030905e84235861cf89"}, + "saxy": {:hex, :saxy, "1.6.0", "02cb4e9bd045f25ac0c70fae8164754878327ee393c338a090288210b02317ee", [:mix], [], "hexpm", "ef42eb4ac983ca77d650fbdb68368b26570f6cc5895f0faa04d34a6f384abad3"}, "sigaws": {:git, "https://github.com/nulib/sigaws.git", "0a754168d7f6f2f8a7bda3b8c6c2a53c0d0139a5", [branch: "otp-24"]}, "sitemapper": {:hex, :sitemapper, "0.9.0", "bdf91162e2dc9e1ca06438a514f21baa6b023e48bccc7a2ca396f648561e083d", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:google_api_storage, "~> 0.34", [hex: :google_api_storage, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "4257fd51381ab9d3b967568eb154a3441bb499a6cd06ad87a06555ff64dff2fd"}, "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"}, diff --git a/app/priv/nodejs/lambda/index.js b/app/priv/nodejs/lambda/index.js index 775c51206..2631d5625 100755 --- a/app/priv/nodejs/lambda/index.js +++ b/app/priv/nodejs/lambda/index.js @@ -7,7 +7,16 @@ const portlog = (level, ...args) => writeln([`[${level}]`, ...args].filter((e) => e != null).join(" ")); ["log", "debug", "info", "warn", "error"].forEach((level) => { - let outLevel = level == "log" ? "info" : level; + let outLevel; + + if (level == "log") { + outLevel = "info"; + } else if (level == "warn") { + outLevel = "warning"; + } else { + outLevel = level; + } + global.console[level] = (...args) => portlog(outLevel, ...args); }); diff --git a/app/test/env_test.exs b/app/test/env_test.exs deleted file mode 100644 index 6e7563aa6..000000000 --- a/app/test/env_test.exs +++ /dev/null @@ -1,67 +0,0 @@ -defmodule EnvTest do - use ExUnit.Case - - import Env - - setup %{environment: environment} do - set_env = fn - {key, nil} -> System.delete_env(key) - {key, value} -> System.put_env(key, value) - end - - saved = Enum.map(environment, fn {key, _} -> {key, System.get_env(key)} end) - Enum.each(environment, set_env) - on_exit(fn -> Enum.each(saved, set_env) end) - end - - describe "hush" do - @describetag environment: [{"SECRETS_PATH", "foo/config"}] - - test "aws_secret/2" do - assert aws_secret("meadow", test: :value) == - {:hush, Hush.Provider.AwsSecretsManager, "foo/config/meadow", [test: :value]} - end - - test "meadow_secret/2" do - assert meadow_secret(test: :value) == - {:hush, Hush.Provider.AwsSecretsManager, "foo/config/meadow", [test: :value]} - end - - test "environment_secret/2" do - assert environment_secret("DEV_PREFIX", test: :value) == - {:hush, Hush.Provider.SystemEnvironment, "DEV_PREFIX", [test: :value]} - end - end - - describe "dev environment" do - @describetag environment: [{"DEV_PREFIX", "env"}] - - test "prefix/0" do - assert prefix() == "env-test" - end - - test "prefix/1" do - assert prefix("database") == "env-test-database" - end - - test "atom_prefix/1" do - assert atom_prefix("database") == :"env-test-database" - end - end - - describe "release environment" do - @describetag environment: [{"DEV_PREFIX", nil}, {"RELEASE_NAME", "meadow"}] - - test "prefix/0" do - assert prefix() == "" - end - - test "prefix/1" do - assert prefix("database") == "database" - end - - test "atom_prefix/1" do - assert atom_prefix("database") == :database - end - end -end diff --git a/app/test/meadow/utils/hush/transformer/cast_test.exs b/app/test/meadow/utils/hush/transformer/cast_test.exs deleted file mode 100644 index 3f304a509..000000000 --- a/app/test/meadow/utils/hush/transformer/cast_test.exs +++ /dev/null @@ -1,59 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.CastTest do - use ExUnit.Case - alias Meadow.Utils.Hush.Transformer.Cast, as: Subject - - test "key/0" do - assert Subject.key() == :cast - end - - describe "transform/2" do - test "nil" do - assert Subject.transform(:atom, nil) == {:ok, nil} - assert Subject.transform(:binary, nil) == {:ok, nil} - assert Subject.transform(:boolean, nil) == {:ok, nil} - assert Subject.transform(:float, nil) == {:ok, nil} - assert Subject.transform(:integer, nil) == {:ok, nil} - end - - test "binary" do - assert Subject.transform(:binary, 123) == {:ok, "123"} - assert Subject.transform(:binary, "value") == {:ok, "value"} - assert Subject.transform(:binary, true) == {:ok, "true"} - assert Subject.transform(:binary, :value) == {:ok, "value"} - end - - test "integer" do - assert Subject.transform(:integer, 123) == {:ok, 123} - assert Subject.transform(:integer, "123") == {:ok, 123} - assert {:error, _} = Subject.transform(:integer, "123.45") - assert {:error, _} = Subject.transform(:integer, :value) - assert {:error, _} = Subject.transform(:integer, "value") - end - - test "boolean" do - assert Subject.transform(:boolean, true) == {:ok, true} - assert Subject.transform(:boolean, false) == {:ok, false} - assert Subject.transform(:boolean, "true") == {:ok, true} - assert Subject.transform(:boolean, "false") == {:ok, false} - assert Subject.transform(:boolean, true) == {:ok, true} - assert Subject.transform(:boolean, false) == {:ok, false} - end - - test "float" do - assert Subject.transform(:float, 123) == {:ok, 123.0} - assert Subject.transform(:float, "123.45") == {:ok, 123.45} - assert {:error, _} = Subject.transform(:float, "123") - assert {:error, _} = Subject.transform(:float, :value) - assert {:error, _} = Subject.transform(:float, "value") - end - - test "atom" do - assert Subject.transform(:atom, :value) == {:ok, :value} - assert Subject.transform(:atom, "value") == {:ok, :value} - assert Subject.transform(:atom, true) == {:ok, true} - assert Subject.transform(:atom, 123.45) == {:ok, :"123.45"} - assert Subject.transform(:atom, 123) == {:ok, :"123"} - assert {:error, _} = Subject.transform(:atom, "fhqwhgads") - end - end -end diff --git a/app/test/meadow/utils/hush/transformer/default_test.exs b/app/test/meadow/utils/hush/transformer/default_test.exs deleted file mode 100644 index 0a420b516..000000000 --- a/app/test/meadow/utils/hush/transformer/default_test.exs +++ /dev/null @@ -1,14 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.DefaultTest do - use ExUnit.Case - alias Meadow.Utils.Hush.Transformer.Default, as: Subject - - test "key/0" do - assert Subject.key() == :default - end - - test "transform/2" do - assert Subject.transform("default", "value") == {:ok, "value"} - assert Subject.transform("default", nil) == {:ok, nil} - assert Subject.transform("default", {:error, :not_found}) == {:ok, "default"} - end -end diff --git a/app/test/meadow/utils/hush/transformer/dig_test.exs b/app/test/meadow/utils/hush/transformer/dig_test.exs deleted file mode 100644 index 73590f108..000000000 --- a/app/test/meadow/utils/hush/transformer/dig_test.exs +++ /dev/null @@ -1,24 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.DigTest do - use ExUnit.Case - alias Meadow.Utils.Hush.Transformer.Dig, as: Subject - - @value %{ - map: %{ - value: 25 - }, - integer: 123, - string: "value" - } - - test "key/0" do - assert Subject.key() == :dig - end - - test "transform/2" do - assert Subject.transform([:map, :value], @value) == {:ok, 25} - assert Subject.transform([:map, :oops], @value) == {:ok, {:error, :not_found}} - assert Subject.transform([:integer], @value) == {:ok, 123} - assert Subject.transform([:string], @value) == {:ok, "value"} - assert Subject.transform([:path], "not accessible") == {:ok, {:error, :not_found}} - end -end diff --git a/app/test/meadow/utils/hush/transformer/split_test.exs b/app/test/meadow/utils/hush/transformer/split_test.exs deleted file mode 100644 index 2d7ca9ccf..000000000 --- a/app/test/meadow/utils/hush/transformer/split_test.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Meadow.Utils.Hush.Transformer.SplitTest do - use ExUnit.Case - alias Meadow.Utils.Hush.Transformer.Split, as: Subject - - test "key/0" do - assert Subject.key() == :split - end - - test "transform/2" do - assert Subject.transform(", ", "1, 2, 3") == {:ok, ["1", "2", " 3"]} - assert Subject.transform(~r/,\s*/, "1,2, 3, 4") == {:ok, ["1", "2", "3", "4"]} - assert Subject.transform({~r/,\s*/, parts: 2}, "1,2, 3, 4") == {:ok, ["1", "2, 3, 4"]} - assert Subject.transform(~r/,\s*/, [1, 2, 3]) == {:ok, [1, 2, 3]} - end -end diff --git a/app/test/pipeline/actions/generate_poster_image_test.exs b/app/test/pipeline/actions/generate_poster_image_test.exs index 2e072e33d..012adad86 100644 --- a/app/test/pipeline/actions/generate_poster_image_test.exs +++ b/app/test/pipeline/actions/generate_poster_image_test.exs @@ -3,11 +3,11 @@ defmodule Meadow.Pipeline.Actions.GeneratePosterImageTest do use Meadow.DataCase use Meadow.PipelineCase + alias Meadow.Config.Secrets alias Meadow.Data.FileSets alias Meadow.Pipeline.Actions.GeneratePosterImage alias Meadow.Utils.Pairtree - import Env import ExUnit.CaptureLog @mediainfo %{ @@ -85,7 +85,7 @@ defmodule Meadow.Pipeline.Actions.GeneratePosterImageTest do assert log |> String.contains?( - "Skipping cache invalidation for: /iiif/3/#{prefix()}/posters/#{file_set_id}/*. No distribution id found." + "Skipping cache invalidation for: /iiif/3/#{Secrets.prefix()}/posters/#{file_set_id}/*. No distribution id found." ) end end diff --git a/app/test/support/bucket_names.ex b/app/test/support/bucket_names.ex index 69d6a7b9c..03e885653 100644 --- a/app/test/support/bucket_names.ex +++ b/app/test/support/bucket_names.ex @@ -13,7 +13,7 @@ defmodule Meadow.BucketNames do @ingest_bucket prefixed.("ingest") @preservation_bucket prefixed.("preservation") @preservation_check_bucket prefixed.("preservation-checks") - @upload_bucket prefixed.("uploads") + @upload_bucket prefixed.("upload") @pyramid_bucket prefixed.("pyramids") @streaming_bucket prefixed.("streaming") end diff --git a/app/test/test_helper.exs b/app/test/test_helper.exs index bc4e36e42..ae1a3b5bd 100644 --- a/app/test/test_helper.exs +++ b/app/test/test_helper.exs @@ -1,4 +1,26 @@ -Hush.resolve!() +alias Meadow.Config.Secrets + +case :ets.info(:secret_cache, :name) do + :secret_cache -> :ets.delete_all_objects(:secret_cache) + :undefined -> :ets.new(:secret_cache, [:set, :protected, :named_table]) +end + +cluster_config = + Application.get_env(:meadow, Meadow.Search.Cluster) + |> Keyword.merge( + url: + Secrets.get_secret( + :meadow, + ["search", "cluster_endpoint"], + "http://localhost:9200" + ), + bulk_page_size: 3, + bulk_wait_interval: 2, + embedding_model_id: nil + ) + +Application.put_env(:meadow, Meadow.Search.Cluster, cluster_config) + Meadow.Repo.wait_for_connection() Mix.Task.run("ecto.setup") diff --git a/infrastructure/deploy/main.tf b/infrastructure/deploy/main.tf index f8a7d5e79..66d3b724a 100644 --- a/infrastructure/deploy/main.tf +++ b/infrastructure/deploy/main.tf @@ -14,6 +14,25 @@ provider "aws" { region = var.aws_region } +locals { +# environment = module.core.outputs.stack.environment + namespace = module.core.outputs.stack.namespace + prefix = module.core.outputs.stack.prefix + tags = merge( + module.core.outputs.stack.tags, + { + Component = "meadow", + Git = "github.com/nulib/meadow" + Project = "Meadow" + } + ) +} + +module "core" { + source = "git::https://github.com/nulib/infrastructure.git//modules/remote_state" + component = "core" +} + module "rds" { source = "terraform-aws-modules/rds/aws" version = "4.1.2" diff --git a/infrastructure/deploy/secrets.tf b/infrastructure/deploy/secrets.tf index 6da51e246..a940ea721 100644 --- a/infrastructure/deploy/secrets.tf +++ b/infrastructure/deploy/secrets.tf @@ -36,32 +36,10 @@ locals { } } - ezid = { - password = var.ezid_password - shoulder = var.ezid_shoulder - target_base_url = var.ezid_target_base_url - url = "https://ezid.cdlib.org/" - user = var.ezid_user - } - geonames = { username = var.geonames_username } - iiif = { - base_url = var.iiif_server_url - distribution_id = var.iiif_cloudfront_distribution_id - manifest_url = var.iiif_manifest_url - } - - search = { - cluster_endpoint = var.elasticsearch_url - access_key_id = aws_iam_access_key.meadow_elasticsearch_access_key.id - secret_access_key = aws_iam_access_key.meadow_elasticsearch_access_key.secret - embedding_model_id = var.embedding_model_id - embedding_dimensions = var.embedding_dimensions - } - ldap = { host = var.ldap_server port = var.ldap_port @@ -75,10 +53,6 @@ locals { role_arn = aws_iam_role.transcode_role.arn } - nusso = { - api_key = var.agentless_sso_key - } - pipeline = { digester = module.pipeline_lambda["digester"].lambda_function_arn exif = module.pipeline_lambda["exif"].lambda_function_arn @@ -104,7 +78,7 @@ locals { } resource "aws_secretsmanager_secret" "config_secrets" { - name = "config/meadow" + name = "${local.prefix}/meadow" description = "Meadow configuration secrets" } diff --git a/infrastructure/localstack/.terraform.lock.hcl b/infrastructure/localstack/.terraform.lock.hcl index 3c2ae0169..c8dfee244 100644 --- a/infrastructure/localstack/.terraform.lock.hcl +++ b/infrastructure/localstack/.terraform.lock.hcl @@ -4,6 +4,7 @@ provider "registry.terraform.io/hashicorp/archive" { version = "2.6.0" hashes = [ + "h1:rYAubRk7UHC/fzYqFV/VHc+7VIY01ugCxauyTYCNf9E=", "h1:upAbF0KeKLAs3UImwwp5veC7jRcLnpKWVjkbd4ziWhM=", "zh:29273484f7423b7c5b3f5df34ccfc53e52bb5e3d7f46a81b65908e7a8fd69072", "zh:3cba58ec3aea5f301caf2acc31e184c55d994cc648126cac39c63ae509a14179", @@ -23,6 +24,7 @@ provider "registry.terraform.io/hashicorp/archive" { provider "registry.terraform.io/hashicorp/aws" { version = "5.70.0" hashes = [ + "h1:LKnWZnujHcQPm3MAk4elP3H9VXNjlO6rNqlO5s330Yg=", "h1:kcKscQCmMLrNMAkaL4XIqGGq4uk8vXthNRvtfersNH0=", "zh:09cbec93c324e6f03a866244ecb2bae71fdf1f5d3d981e858b745c90606b6b6d", "zh:19685d9f4c9ddcfa476a9a428c6c612be4a1b4e8e1198fbcbb76436b735284ee", @@ -45,6 +47,7 @@ provider "registry.terraform.io/hashicorp/aws" { provider "registry.terraform.io/hashicorp/null" { version = "3.2.3" hashes = [ + "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", diff --git a/infrastructure/localstack/docker-compose.yml b/infrastructure/localstack/docker-compose.yml new file mode 100644 index 000000000..ff9dfb472 --- /dev/null +++ b/infrastructure/localstack/docker-compose.yml @@ -0,0 +1,38 @@ +--- +services: + db: + image: ghcr.io/nulib/postgres:10-alpine + environment: + POSTGRES_USER: docker + POSTGRES_PASSWORD: d0ck3r + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 10s + timeout: 5s + retries: 5 + ports: + - 5432:5432 + ldap: + image: ghcr.io/nulib/ldap-alpine + ports: + - 389:389 + - 636:636 + opensearch: + image: opensearchproject/opensearch:2.11.1 + environment: + bootstrap.memory_lock: true + OPENSEARCH_JAVA_OPTS: "-Xms256m -Xmx256m" + DISABLE_INSTALL_DEMO_CONFIG: true + DISABLE_SECURITY_PLUGIN: true + discovery.type: single-node + ports: + - 9200:9200 + localstack: + image: localstack/localstack + environment: + DOCKER_HOST: unix:///var/run/docker.sock + GATEWAY_LISTEN: 0.0.0.0:4566 + ports: + - 4566:4566 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro \ No newline at end of file diff --git a/infrastructure/localstack/environment_config.tf b/infrastructure/localstack/environment_config.tf index e28d3d60a..d17f192b3 100644 --- a/infrastructure/localstack/environment_config.tf +++ b/infrastructure/localstack/environment_config.tf @@ -1,51 +1,112 @@ locals { - project = "meadow" - port_offset = 0 # terraform.workspace == "test" ? 2 : 1 - - computed_secrets = { - db = { - host = "localhost" - port = 5432 + local.port_offset - user = "docker" - password = "d0ck3r" + secrets = { + "config/meadow" = { + buckets = { + ingest = "test-ingest" + preservation = "test-preservation" + preservation_check = "test-preservation-checks" + pyramid = "test-pyramids" + sitemap = "test-sitemaps" + streaming = "test-streaming" + upload = "test-upload" + } + + db = { + database = "postgres" + host = "localhost" + port = 5432 + user = "docker" + password = "d0ck3r" + } + + dc = { + base_url = "https://dc.dev.library.northwestern.edu/" + } + + dc_api = { + v2 = { + api_token_secret = "TEST_SECRET" + api_token_ttl = 300 + base_url = "http://dcapi-test.northwestern.edu" + } + iiif_distribution_id = null + } + + geonames = { + username = "nul_rdc" + } + + iiif = { + base_url = "http://localhost:8184/iiif/3/" + distribution_id = null + manifest_url = "http://test-pyramids.s3.localhost.localstack.cloud:4566/public/" + } + + mediaconvert = { + queue = "arn:aws:mediaconvert:::queues/Default" + role_arn = "arn:aws:iam:::role/service-role/MediaConvert_Default_Role" + } + + streaming = { + base_url = "https://test-streaming-url/" + distribution_id = "Z7Q9N4L3X8P5J2" + } + + work_archiver = { + endpoint = null + } + } - index = { - index_endpoint = "http://localhost:${9200 + local.port_offset}" - kibana_endpoint = "http://localhost:${5601 + local.port_offset}" + "infrastructure/ezid" = { + password = "mockpassword" + shoulder = "ark:/12345/nu1" + target_base_url = "https://devbox.library.northwestern.edu:3333/items/" + url = "http://localhost:3944" + user = "mockuser" } - ldap = { - host = "localhost" - base = "DC=library,DC=northwestern,DC=edu" - port = 389 + local.port_offset - user_dn = "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu" - password = "d0ck3rAdm1n!" - ssl = "false" + + "infrastructure/iiif" = { + base = "http://localhost/" + v2 = "http://localhost/iiif/2" + v3 = "http://localhost/iiif/3" } - } - config_secrets = merge(var.config_secrets, local.computed_secrets) -} + "infrastructure/index" = { + endpoint = "http://localhost:9200" + } -resource "aws_secretsmanager_secret" "config_secrets" { - name = "config/meadow" - description = "Meadow configuration secrets" -} + # "infrastructure/inference" = { + # endpoints = { + # endpoint = "https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-multilingual-v3/invoke" + # name = "cohere.embed-multilingual-v3" + # } + + # } + + "infrastructure/ldap" = { + base = "OU=test,DC=library,DC=northwestern,DC=edu" + host = "localhost" + port = 389 + user_dn = "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu" + password = "d0ck3rAdm1n!" + ssl = false + } -resource "aws_secretsmanager_secret" "ssl_certificate" { - name = "config/wildcard_ssl" - description = "Wildcard SSL certificate and private key" + "infrastructure/nusso" = { + api_key = "test-sso-key" + base_url = "https://northwestern-dev.apigee.net/agentless-websso/" + } + } } -resource "aws_secretsmanager_secret_version" "config_secrets" { - secret_id = aws_secretsmanager_secret.config_secrets.id - secret_string = jsonencode(local.config_secrets) +resource "aws_secretsmanager_secret" "config_secrets" { + for_each = local.secrets + name = each.key } -resource "aws_secretsmanager_secret_version" "ssl_certificate" { - secret_id = aws_secretsmanager_secret.ssl_certificate.id - secret_string = jsonencode({ - certificate = file(var.ssl_certificate_file) - key = file(var.ssl_key_file) - }) +resource "aws_secretsmanager_secret_version" "config_secrets" { + for_each = local.secrets + secret_id = aws_secretsmanager_secret.config_secrets[each.key].id + secret_string = jsonencode(each.value) } diff --git a/infrastructure/localstack/test.tfvars b/infrastructure/localstack/test.tfvars index 0e7a7c1b2..886898c51 100644 --- a/infrastructure/localstack/test.tfvars +++ b/infrastructure/localstack/test.tfvars @@ -1,27 +1,2 @@ -config_secrets = { - ezid = { - password = "apitest" - shoulder = "ark:/99999/fk4" - user = "apitest" - } - - geonames = { - username = "" - } - - iiif = { - manifest_url = "http://test-pyramids.s3.localhost.localstack.cloud:4568/public/" - } - - nusso = { - api_key = "test-sso-key" - base_url = "https://northwestern-dev.apigee.net/agentless-websso/" - } - - streaming = { - base_url = "https://test-streaming-url/" - } -} - ssl_certificate_file = "/dev/null" ssl_key_file = "/dev/null" diff --git a/infrastructure/localstack/variables.tf b/infrastructure/localstack/variables.tf index df51a63be..3ab5dd271 100644 --- a/infrastructure/localstack/variables.tf +++ b/infrastructure/localstack/variables.tf @@ -1,8 +1,3 @@ -variable "config_secrets" { - type = map(map(string)) - default = {} -} - variable "ssl_certificate_file" { type = string default = "../../miscellany/devbox_cert/dev.rdc.wildcard.full.pem"