Improve uncaught error handling

This reworks the production system to:

- Exit on an uncaught exception. This will allow systemd to restart the
  application when an uncaught error occurs instead of it continuing to
  run with a dead thread
- Attempt to log the uncaught exception locally as well
- Don't let an exception when reporting the uncaught exception prevent
  the exit from occurring
- Exit when an error occurs at startup

src/clojars/errors.clj

@@ -37,14 +37,7 @@
                         id
                         (or (:message extra) "RavenErrorReporter capture")
                         e
-                        extra))
-
-  Thread$UncaughtExceptionHandler
-  (uncaughtException [_this _thread throwable]
-    (raven-error-report (:dsn raven-config)
-                        nil
-                        "UncaughtExceptionHandler capture"
-                        throwable)))
+                        extra)))
 
 (defn raven-error-reporter [raven-config]
   (->RavenErrorReporter raven-config))
@@ -68,13 +61,6 @@
 (defn stdout-reporter []
   (->StdOutReporter))
 
-(defrecord NullReporter []
-  ErrorReporter
-  (-report-error [_ _ _ _]))
-
-(defn null-reporter []
-  (->NullReporter))
-
 (defrecord MultiReporter [reporters]
   ErrorReporter
   (-report-error [_reporter e extra id]
@@ -111,3 +97,16 @@
                                   error-page/error-page-response)]
             (->> (report-ring-error reporter t req request-id)
                  (err-response-fn (ex-data t)))))))))
+
+(defn- uncaugt-exception-handler
+  [error-reporter]
+  (reify Thread$UncaughtExceptionHandler
+    (uncaughtException [_ _thread ex]
+      (try
+        (report-error error-reporter ex)
+        (finally
+          (System/exit 70))))))
+
+(defn set-default-exception-handler
+  [error-reporter]
+  (Thread/setDefaultUncaughtExceptionHandler (uncaugt-exception-handler error-reporter)))

src/clojars/main.clj

@@ -10,44 +10,48 @@
    [meta-merge.core :refer [meta-merge]]
    [raven-clj.core :as raven-clj]))
 
-(def prod-env
+(def ^:private prod-env
   {:app {:middleware []}})
 
-(defn info [& msg]
+(defn- info [& msg]
   (apply println "clojars-web:" msg))
 
-(defn warn [& msg]
+(defn- warn [& msg]
   (apply println "clojars-web: WARNING -" msg))
 
-(defn prod-system [config prod-reporter]
+(defn- prod-system [config prod-reporter]
   (-> (meta-merge config prod-env)
       system/new-system
-      (assoc
-       :error-reporter (err/multiple-reporters
-                        (err/log-reporter)
-                        prod-reporter))))
+      (assoc :error-reporter prod-reporter)))
 
-(defn error-reporter [config]
-  (let [dsn (:sentry-dsn config)]
-    (if dsn
-      (let [raven-reporter (err/raven-error-reporter {:dsn dsn})]
-        (info "enabling raven-clj client dsn:project-id:" (:project-id (raven-clj/parse-dsn dsn)))
-        (Thread/setDefaultUncaughtExceptionHandler raven-reporter)
-        raven-reporter)
-      (do
-        (warn "no :sentry-dsn set in config, errors won't be logged to Sentry")
-        (err/null-reporter)))))
+(defn- raven-reporter
+  [{:as _config :keys [sentry-dsn]}]
+  (if sentry-dsn
+    (let [raven-reporter (err/raven-error-reporter {:dsn sentry-dsn})]
+      (info "enabling raven-clj client dsn:project-id:" (:project-id (raven-clj/parse-dsn sentry-dsn)))
+      raven-reporter)
+    (warn "no :sentry-dsn set in config, errors won't be logged to Sentry")))
+
+(defn- error-reporter [config]
+  (if-some [raven-reporter (raven-reporter config)]
+    (err/multiple-reporters
+     (err/log-reporter)
+     raven-reporter)
+    (err/log-reporter)))
 
 (defn -main [& _args]
   (try
     (alter-var-root #'config/*profile* (constantly "production"))
     (let [config (config/config)
-          system (component/start (prod-system config (error-reporter config)))]
+          error-reporter (error-reporter config)
+          system (component/start (prod-system config error-reporter))]
+      (err/set-default-exception-handler error-reporter)
       (info "starting jetty on" (str "http://" (:bind config) ":" (:port config)))
       (admin/init (get-in system [:db :spec])
                   (:search system)
                   (:storage system)))
     (catch Throwable t
       (binding [*out* *err*]
         (println "Error during app startup:"))
-      (.printStackTrace t))))
+      (.printStackTrace t)
+      (System/exit 70))))