Skip to content

Commit

Permalink
Move all dynamic secrets to Meadow.Config.Runtime namespace
Browse files Browse the repository at this point in the history
Remove janky `Env` module and `Hush` dependency

Remove Hush from test_helper.exs

Consolidate component configurations into compile-time/runtime

Push more test configs into localstack secrets manager

Turn off warnings on lambda logging
  • Loading branch information
mbklein committed Nov 6, 2024
1 parent 80f4e55 commit 09f5bb5
Show file tree
Hide file tree
Showing 42 changed files with 978 additions and 1,294 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ jobs:
POSTGRES_USER: docker
POSTGRES_PASSWORD: d0ck3r
options: >-
--health-cmd "pg_isready -U postgres"
--health-cmd "pg_isready -U docker"
--health-interval 10s
--health-timeout 5s
--health-retries 5
Expand Down Expand Up @@ -221,6 +221,8 @@ jobs:
restore-keys: |
${{ runner.os }}-npm-v6-${{ hashFiles('**/package-lock.json') }}
${{ runner.os }}-npm-v6-
- name: Set SECRET_KEY_BASE
run: echo "SECRET_KEY_BASE=$(openssl rand -hex 32)" >> $GITHUB_ENV
- name: Elixir Static Analysis
run: mix credo
working-directory: app
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ npm-debug.log
*.tfvars

/infrastructure/**/.terraform/
/infrastructure/**/*.tfstate*
/infrastructure/**/_build/
/infrastructure/**/build/
/infrastructure/**/builds/
Expand Down
2 changes: 1 addition & 1 deletion .tool-versions
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
nodejs 18.18.0
erlang 26.0.2
elixir 1.15.4-otp-26
terraform 1.1.5
terraform 1.7.4
tflint 0.34.1
#npm 10.1.0
1 change: 1 addition & 0 deletions app/config/.credo.exs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
excluded: []
},
checks: [
{Credo.Check.Design.TagTODO, false},
{Credo.Check.Refactor.MapInto, false},
{Credo.Check.Warning.LazyLogging, false}
]
Expand Down
283 changes: 51 additions & 232 deletions app/config/config.exs
Original file line number Diff line number Diff line change
@@ -1,207 +1,44 @@
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
# Meadow configuration takes place across multiple files.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
# Compile-time configs are evaluated when Meadow is compiled, with
# higher priority given to files farther down the list (i.e., environment-
# specific configs take precedence over global config).
#
# config/config.exs
# config/[environment].exs
#
# Runtime configs work the same way, except they are set at runtime.
# This means they can read configuration data that may not be available
# at compile-time, or that may change. This includes files on disk,
# the runtime system environment, or remote datastores such as AWS
# Secrets Manager.
#
# lib/meadow/config/runtime.ex
# lib/meadow/config/runtime/[environment].ex
# config/[environment].local.exs
#
# When one configuration takes precedence over other, keyword lists are
# deep merged, while other values (e.g., maps) are overwritten.
#
# It is _strongly_ suggested that component configurations be done entirely
# at compile-time _or_ runtime and not some of each, e.g.:
# don't try to
# ```
# config :meadow, :component, [static properties]
# ```
# at compile time and
# ```
# config :meadow, :component, [dynamic properties]
# ```
# at runtime.

# General application configuration
import Config
import Env

alias Meadow.Utils.Hush.Transformer, as: CustomTransformer
alias Meadow.Pipeline.Actions

config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase

config :meadow,
ecto_repos: [Meadow.Repo],
environment: Mix.env(),
environment_prefix: prefix()

config :meadow, Meadow.Repo,
migration_primary_key: [
name: :id,
type: :binary_id,
autogenerate: false,
read_after_writes: true,
default: {:fragment, "uuid_generate_v4()"}
],
migration_timestamps: [type: :utc_datetime_usec],
username: aws_secret("meadow", dig: ["db", "user"], default: "docker"),
password: aws_secret("meadow", dig: ["db", "password"], default: "d0ck3r"),
database: aws_secret("meadow", dig: ["db", "database"], default: prefix("meadow")),
hostname: aws_secret("meadow", dig: ["db", "host"], default: "localhost"),
port: aws_secret("meadow", dig: ["db", "port"], default: 5432)

# Configures the endpoint
config :meadow, MeadowWeb.Endpoint,
url: [host: "localhost"],
secret_key_base: "C7BC/yBsTCe/PaJ9g0krwlQrNZZV2r3jSjeuGCeIu9mfNE+4bPcNPHiINQtIQk/B",
render_errors: [view: MeadowWeb.ErrorView, accepts: ~w(html json)],
pubsub_server: Meadow.PubSub,
live_view: [signing_salt: "C7BC/yBsTCe/PaJ9g0krwlQrNZZV2r3jSjeuGCeIu9mfNE+4bPcNPHiINQtIQk/B"]

# Configures the Search cluster
config :meadow, Meadow.Search.Cluster,
url:
aws_secret("meadow",
dig: ["search", "cluster_endpoint"],
default: "http://localhost:9201"
),
indexes: [
%{
name: prefix("dc-v2-work"),
settings: "priv/search/v2/settings/work.json",
version: 2,
schemas: [Meadow.Data.Schemas.Work],
pipeline: prefix("dc-v2-work-pipeline")
},
%{
name: prefix("dc-v2-file-set"),
settings: "priv/search/v2/settings/file_set.json",
version: 2,
schemas: [Meadow.Data.Schemas.FileSet]
},
%{
name: prefix("dc-v2-collection"),
settings: "priv/search/v2/settings/collection.json",
version: 2,
schemas: [Meadow.Data.Schemas.Collection]
}
],
default_options: [
timeout: 20_000,
recv_timeout: 30_000
],
bulk_page_size: 200,
bulk_wait_interval: 500,
json_encoder: Ecto.Jason,
embedding_model_id:
aws_secret("meadow",
dig: ["search", "embedding_model_id"],
default: nil
),
embedding_dimensions:
aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil),
embedding_text_fields: [
:title,
:alternate_title,
:description,
:collection,
:creator,
:contributor,
:date_created,
:genre,
:subject,
:style_period,
:language,
:location,
:publisher,
:technique,
:physical_description_material,
:physical_description_size,
:caption,
:table_of_contents,
:scope_and_contents,
:abstract,
]

config :meadow,
ark: %{
default_shoulder: aws_secret("meadow", dig: ["ezid", "shoulder"], default: "ark:/12345/nu1"),
user: aws_secret("meadow", dig: ["ezid", "user"], default: "ark_user"),
password: aws_secret("meadow", dig: ["ezid", "password"], default: "ark_password"),
target_url:
aws_secret("meadow",
dig: ["ezid", "target_base_url"],
default: "https://devbox.library.northwestern.edu:3333/items/"
),
url: aws_secret("meadow", dig: ["ezid", "url"], default: "http://localhost:3943")
},
ingest_bucket: prefix("ingest"),
upload_bucket: prefix("uploads"),
pipeline_delay: :timer.seconds(5),
preservation_bucket: prefix("preservation"),
preservation_check_bucket: prefix("preservation-checks"),
pyramid_bucket: prefix("pyramids"),
streaming_bucket: prefix("streaming"),
streaming_url:
aws_secret("meadow",
dig: ["streaming", "base_url"],
default: "https://#{prefix()}-streaming.s3.amazonaws.com/"
),
mediaconvert_client: MediaConvert,
multipart_upload_concurrency: System.get_env("MULTIPART_UPLOAD_CONCURRENCY", "10"),
iiif_server_url:
aws_secret("meadow",
dig: ["iiif", "base_url"],
default: "https://iiif.dev.rdc.library.northwestern.edu/iiif/3/#{prefix()}"
),
iiif_manifest_url_deprecated:
aws_secret("meadow",
dig: ["iiif", "manifest_url"],
default: "https://#{prefix()}-pyramids.s3.amazonaws.com/public/"
),
iiif_distribution_id: aws_secret("meadow", dig: ["iiif", "distribution_id"], default: nil),
digital_collections_url:
aws_secret("meadow",
dig: ["dc", "base_url"],
default: "https://dc.rdc-staging.library.northwestern.edu/"
),
progress_ping_interval: System.get_env("PROGRESS_PING_INTERVAL", "1000"),
validation_ping_interval: System.get_env("VALIDATION_PING_INTERVAL", "1000"),
shared_links_index: prefix("shared_links"),
pyramid_tiff_working_dir: System.tmp_dir!(),
streaming_distribution_id:
aws_secret("meadow", dig: ["streaming", "distribution_id"], default: nil),
work_archiver_endpoint: aws_secret("meadow", dig: ["work_archiver", "endpoint"], default: "")

config :exldap, :settings,
server: aws_secret("meadow", dig: ["ldap", "host"], default: "localhost"),
base:
aws_secret("meadow",
dig: ["ldap", "base"],
default: "DC=library,DC=northwestern,DC=edu"
),
port: aws_secret("meadow", dig: ["ldap", "port"], cast: :integer, default: 390),
user_dn:
aws_secret("meadow",
dig: ["ldap", "user_dn"],
default: "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu"
),
password: aws_secret("meadow", dig: ["ldap", "password"], default: "d0ck3rAdm1n!"),
ssl: aws_secret("meadow", dig: ["ldap", "ssl"], cast: :boolean, default: false)

config :meadow,
transcoding_presets: %{
audio: [
%{NameModifier: "-high", Preset: "meadow-audio-high"},
%{NameModifier: "-medium", Preset: "meadow-audio-medium"}
],
video: [
%{NameModifier: "-1080", Preset: "meadow-video-high"},
%{NameModifier: "-720", Preset: "meadow-video-medium"},
%{NameModifier: "-540", Preset: "meadow-video-low"}
]
}

# Configure checksum requirements
config :meadow,
required_checksum_tags: ["computed-md5"],
checksum_wait_timeout: 3_600_000

# Configure Lambda-based actions
lambda_from_ssm = fn lambda, function ->
{:lambda, aws_secret("meadow", dig: ["pipeline", lambda], default: "#{function}:$LATEST")}
end

config :meadow, :lambda,
digester: lambda_from_ssm.("digester", "digester"),
exif: lambda_from_ssm.("exif", "exif"),
frame_extractor: lambda_from_ssm.("frame_extractor", "frame-extractor"),
mediainfo: lambda_from_ssm.("mediainfo", "mediainfo"),
mime_type: lambda_from_ssm.("mime_type", "mime-type"),
tiff: lambda_from_ssm.("tiff", "pyramid-tiff")

# Configures Elixir's Logger
config :logger, :console,
format: "$time $metadata[$level] $message\n",
Expand All @@ -210,8 +47,6 @@ config :logger, :console,
# Use Jason for JSON parsing in Phoenix
config :phoenix, :json_library, Jason

config :ueberauth, Ueberauth, providers: [nusso: {Ueberauth.Strategy.NuSSO, []}]

config :authoritex,
authorities: [
Authoritex.FAST.CorporateName,
Expand All @@ -235,20 +70,6 @@ config :authoritex,
NUL.Authority
]

config :honeybadger,
api_key: environment_secret("HONEYBADGER_API_KEY", default: "DO_NOT_REPORT"),
environment_name: System.get_env("HONEYBADGER_ENVIRONMENT", to_string(Mix.env())),
revision: System.get_env("HONEYBADGER_REVISION", ""),
repos: [Meadow.Repo],
breadcrumbs_enabled: true,
filter: Meadow.Error.Filter,
exclude_envs: [:dev, :test]

config :ex_aws,
access_key_id: [:instance_role],
secret_access_key: [:instance_role],
region: System.get_env("AWS_REGION", "us-east-1")

config :httpoison_retry, wait: 50

config :meadow, :extra_mime_types, %{
Expand All @@ -266,28 +87,26 @@ config :meadow, :extra_mime_types, %{
"xml" => "application/xml"
}

config :meadow,
dc_api: [
v2: aws_secret("meadow", dig: ["dc_api", "v2"])
config :meadow, Meadow.Pipeline,
actions: [
Actions.IngestFileSet,
Actions.ExtractMimeType,
Actions.InitializeDispatch,
Actions.GenerateFileSetDigests,
Actions.ExtractExifMetadata,
Actions.CopyFileToPreservation,
Actions.CreateDerivativeCopy,
Actions.CreatePyramidTiff,
Actions.ExtractMediaMetadata,
Actions.CreateTranscodeJob,
Actions.TranscodeComplete,
Actions.GeneratePosterImage,
Actions.FileSetComplete
]

config :hush,
transformers_override: true,
transformers: [
CustomTransformer.Dig,
CustomTransformer.Default,
CustomTransformer.Split,
Hush.Transformer.Apply,
CustomTransformer.Cast,
Hush.Transformer.ToFile
config :ueberauth, Ueberauth,
providers: [
nusso: {Ueberauth.Strategy.NuSSO, []}
]

import_config "pipeline.exs"

# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.

import_config("#{Mix.env()}.exs")

if File.exists?("config/#{Mix.env()}.local.exs"),
do: import_config("#{Mix.env()}.local.exs")
Loading

0 comments on commit 09f5bb5

Please sign in to comment.