feat: make Logflare.PubSub :pool_size configurable (#1922)

* feat: make Logflare.PubSub :pool_size configurable * chore: set prod pubsub pool size via instance template * fix: convert string value of LOGFLARE_PUBSUB_POOL_SIZE to integer
Logflare · Feb 5, 2024 · a61130a · a61130a
1 parent 72a7a3c
commit a61130a
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 11 deletions.
diff --git a/cloudbuild/prod/pre-deploy.yaml b/cloudbuild/prod/pre-deploy.yaml
@@ -22,7 +22,7 @@ steps:
       - --container-image=${_CONTAINER_IMAGE}
       - --container-privileged
       - --container-restart-policy=always
-      - --container-env=LOGFLARE_GRPC_PORT=4001,LOGFLARE_MIN_CLUSTER_SIZE=2,RELEASE_COOKIE=${_COOKIE},LOGFLARE_LOGGER_METADATA_CLUSTER=${_CLUSTER}
+      - --container-env=LOGFLARE_GRPC_PORT=4001,LOGFLARE_MIN_CLUSTER_SIZE=2,RELEASE_COOKIE=${_COOKIE},LOGFLARE_PUBSUB_POOL_SIZE=56,LOGFLARE_LOGGER_METADATA_CLUSTER=${_CLUSTER}
       - --no-shielded-secure-boot
       - --shielded-vtpm
       - --shielded-integrity-monitoring

diff --git a/config/runtime.exs b/config/runtime.exs
@@ -4,6 +4,17 @@ filter_nil_kv_pairs = fn pairs when is_list(pairs) ->
   Enum.filter(pairs, fn {_k, v} -> v !== nil end)
 end
 
+config :logflare,
+       Logflare.PubSub,
+       [
+         pool_size:
+           if(System.get_env("LOGFLARE_PUBSUB_POOL_SIZE") != nil,
+             do: String.to_integer(System.get_env("LOGFLARE_PUBSUB_POOL_SIZE")),
+             else: nil
+           )
+       ]
+       |> filter_nil_kv_pairs.()
+
 config :logflare,
        [
          node_shutdown_code: System.get_env("LOGFLARE_NODE_SHUTDOWN_CODE"),

diff --git a/docs/docs.logflare.com/docs/self-hosting/index.md b/docs/docs.logflare.com/docs/self-hosting/index.md
@@ -19,16 +19,17 @@ All browser authentication will be disabled when in single-tenant mode.
 
 ### Common Configuration
 
-| Env Var                            | Type                                                                      | Description                                                                                                                 |
-| ---------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
-| `LOGFLARE_SINGLE_TENANT`           | Boolean, defaults to `false`                                              | If enabled, a singular user will be seeded. All browser usage will default to the user.                                     |
-| `LOGFLARE_API_KEY`                 | string, defaults to `nil`                                                 | If set, this API Key can be used for interacting with the Logflare API. API key will be automatically generated if not set. |
-| `LOGFLARE_SUPABASE_MODE`           | Boolean, defaults to `false`                                              | A special mode for Logflare, where Supabase-specific resources will be seeded. Intended for Suapbase self-hosted usage.     |
-| `PHX_HTTP_PORT`                    | Integer, defaults to `4000`                                               | Allows configuration of the HTTP server port.                                                                               |
-| `DB_SCHEMA`                        | String, defaults to `nil`                                                 | Allows configuration of the database schema to scope Logflare operations.                                                   |
-| `LOGFLARE_LOG_LEVEL`               | String, defaults to `info`. <br/>Options: `error`,`warn`, `info`, `debug` | Allows runtime configuration of log level.                                                                                  |
-| `LOGFLARE_NODE_HOST`               | string, defaults to `127.0.0.1`                                           | Sets node host on startup, which affects the node name `logflare@<host>`                                                    |
-| `LOGFLARE_LOGGER_METADATA_CLUSTER` | string, defaults to `nil`                                                 | Sets global logging metadata for the cluster name. Useful for filtering logs by cluster name.                               |
+| Env Var                            | Type                                                                      | Description                                                                                                                                                      |
+| ---------------------------------- | ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `LOGFLARE_SINGLE_TENANT`           | Boolean, defaults to `false`                                              | If enabled, a singular user will be seeded. All browser usage will default to the user.                                                                          |
+| `LOGFLARE_API_KEY`                 | string, defaults to `nil`                                                 | If set, this API Key can be used for interacting with the Logflare API. API key will be automatically generated if not set.                                      |
+| `LOGFLARE_SUPABASE_MODE`           | Boolean, defaults to `false`                                              | A special mode for Logflare, where Supabase-specific resources will be seeded. Intended for Suapbase self-hosted usage.                                          |
+| `PHX_HTTP_PORT`                    | Integer, defaults to `4000`                                               | Allows configuration of the HTTP server port.                                                                                                                    |
+| `DB_SCHEMA`                        | String, defaults to `nil`                                                 | Allows configuration of the database schema to scope Logflare operations.                                                                                        |
+| `LOGFLARE_LOG_LEVEL`               | String, defaults to `info`. <br/>Options: `error`,`warn`, `info`, `debug` | Allows runtime configuration of log level.                                                                                                                       |
+| `LOGFLARE_NODE_HOST`               | string, defaults to `127.0.0.1`                                           | Sets node host on startup, which affects the node name `logflare@<host>`                                                                                         |
+| `LOGFLARE_LOGGER_METADATA_CLUSTER` | string, defaults to `nil`                                                 | Sets global logging metadata for the cluster name. Useful for filtering logs by cluster name.                                                                    |
+| `LOGFLARE_PUBSUB_POOL_SIZE`        | Integer, defaults to `10`                                                 | Sets the number of `Phoenix.PubSub.PG2` partitions to be created. Should be configured to the number of cores of your server for optimal multi-node performance. |
 
 ### BigQuery Backend Configuration