From d2a1ee71bd345f16ae50915f5e35318fb0fe863c Mon Sep 17 00:00:00 2001 From: Toby Crawley Date: Mon, 3 Jul 2023 09:12:58 -0400 Subject: [PATCH] Remove repo listing route This is no longer used since we pre-compute the indexes and write them to s3 on deploy. --- src/clojars/repo_indexing.clj | 8 +- src/clojars/routes/repo_listing.clj | 14 --- src/clojars/system.clj | 5 +- src/clojars/web.clj | 4 +- src/clojars/web/repo_listing.clj | 94 --------------------- test/clojars/unit/web/repo_listing_test.clj | 40 --------- 6 files changed, 6 insertions(+), 159 deletions(-) delete mode 100644 src/clojars/routes/repo_listing.clj delete mode 100644 src/clojars/web/repo_listing.clj delete mode 100644 test/clojars/unit/web/repo_listing_test.clj diff --git a/src/clojars/repo_indexing.clj b/src/clojars/repo_indexing.clj index 71a60100..c05975a2 100644 --- a/src/clojars/repo_indexing.clj +++ b/src/clojars/repo_indexing.clj @@ -61,7 +61,7 @@ size "\n"))) -(defn generate-index +(defn- generate-index ^String [path entries] (safe-hiccup/html5 @@ -96,7 +96,7 @@ (def ^:private index-file-name "index.html") -(defn get-path-entries +(defn- get-path-entries [repo-bucket path] (->> (s3/list-entries repo-bucket path) ;; filter out index files @@ -105,7 +105,7 @@ (str/ends-with? Key index-file-name)))) (sort-entries))) -(defn normalize-path +(defn- normalize-path [path] (let [path (cond (str/blank? path) nil @@ -117,7 +117,7 @@ (str path "/") path))) -(defn handler +(defn- handler [{:keys [error-reporter repo-bucket]} type {:as _data :keys [path]}] (when (= :repo-path-needs-index type) (let [path (normalize-path path)] diff --git a/src/clojars/routes/repo_listing.clj b/src/clojars/routes/repo_listing.clj deleted file mode 100644 index 8e56e77a..00000000 --- a/src/clojars/routes/repo_listing.clj +++ /dev/null @@ -1,14 +0,0 @@ -(ns clojars.routes.repo-listing - (:require - [clojars.web.repo-listing :as repo-listing] - [compojure.core :as compojure :refer [GET HEAD]])) - -(defn routes - [repo-lister] - (compojure/routes - (GET ["/list-repo"] - {{:keys [path]} :params} - (repo-listing/index-for-path repo-lister path)) - (HEAD ["/list-repo"] - {{:keys [path]} :params} - (repo-listing/index-for-path repo-lister path)))) diff --git a/src/clojars/system.clj b/src/clojars/system.clj index 2b2470b6..3b9b825b 100644 --- a/src/clojars/system.clj +++ b/src/clojars/system.clj @@ -19,7 +19,6 @@ [clojars.stats :refer [artifact-stats]] [clojars.storage :as storage] [clojars.web :as web] - [clojars.web.repo-listing :as repo-listing] [com.stuartsierra.component :as component] [duct.component.endpoint :refer [endpoint-component]] [duct.component.handler :refer [handler-component]] @@ -93,16 +92,14 @@ :notifications (notifications/notification-component) :repo-bucket (s3/s3-client (get-in config [:s3 :repo-bucket])) :repo-indexer (repo-indexing/repo-indexing-component) - :repo-lister (repo-listing/repo-lister (:cache-path config)) :storage (storage-component (:repo config) (:cdn-token config) (:cdn-url config)))) (component/system-using {:app [:clojars-app] :clojars-app [:db :github :gitlab :error-reporter :event-emitter :http-client - :mailer :repo-lister :stats :search :storage] + :mailer :stats :search :storage] :event-emitter [:error-reporter] :http [:app] :notifications [:db :mailer] :repo-indexer [:error-reporter :repo-bucket] - :repo-lister [:repo-bucket] :event-receiver [:error-reporter] :storage [:error-reporter :repo-bucket]})))) diff --git a/src/clojars/web.clj b/src/clojars/web.clj index c2b1a36d..d65b24fe 100644 --- a/src/clojars/web.clj +++ b/src/clojars/web.clj @@ -15,7 +15,6 @@ [clojars.routes.artifact :as artifact] [clojars.routes.group :as group] [clojars.routes.repo :as repo] - [clojars.routes.repo-listing :as repo-listing] [clojars.routes.session :as session] [clojars.routes.token :as token] [clojars.routes.token-breach :as token-breach] @@ -48,7 +47,7 @@ :status 400}))))) (defn- main-routes - [{:as _system :keys [db event-emitter mailer repo-lister search stats]}] + [{:as _system :keys [db event-emitter mailer search stats]}] (let [db (:spec db)] (routes (GET "/" _ @@ -77,7 +76,6 @@ #(html-doc "DMCA" {:account %} (raw (slurp (io/resource "dmca.html")))))) session/routes - (repo-listing/routes repo-lister) (group/routes db event-emitter) (artifact/routes db stats) ;; user routes must go after artifact routes diff --git a/src/clojars/web/repo_listing.clj b/src/clojars/web/repo_listing.clj deleted file mode 100644 index ebf1d040..00000000 --- a/src/clojars/web/repo_listing.clj +++ /dev/null @@ -1,94 +0,0 @@ -(ns clojars.web.repo-listing - (:require - [clojars.maven :as maven] - [clojars.repo-indexing :as repo-indexing] - [clojars.web.safe-hiccup :as safe-hiccup] - [clojure.edn :as edn] - [clojure.java.io :as io] - [ring.util.response :as ring.response]) - (:import - (java.io - File) - (java.util - Date))) - -(set! *warn-on-reflection* true) - -(def ^:private max-age 43200) ;; 12 hours - -;; Public for use in tests -(defn cache-file - ^File - [cache-path path] - ;; assumes path has gone through normalize-path already - (let [path (if path - ;; strip trailing / - (subs path 0 (dec (count path))) - "root") - f (io/file (format "%s/%s.edn" cache-path path))] - (io/make-parents f) - f)) - -(defn- file-age - [^File f] - (-> (- (.getTime (Date.)) (.lastModified f)) - (/ 1000) - (long))) - -(def ^:private not-found-response - (-> (safe-hiccup/html5 - {:lang "en"} - [:head [:title "404 Not Found"]] - [:body [:h1 "404 Not Found"]]) - (ring.response/not-found) - (ring.response/content-type "text/html;charset=utf-8"))) - -(defn- get-cached-response - [cache-path requested-path] - (try - (let [cached-file (cache-file cache-path requested-path)] - (when (.exists cached-file) - (let [age (file-age cached-file)] - (when-not (> age max-age) - (let [response (edn/read-string (slurp cached-file))] - ;; We don't cache the full :not-found response since it is static to - ;; save disk space - [(if (= :not-found response) not-found-response response) age]))))) - (catch Exception _))) - -(defn- cache-response - [cache-path requested-path response] - (spit (cache-file cache-path requested-path) - (binding [*print-length* nil] - (pr-str (if (= not-found-response response) :not-found response)))) - [response 0]) - -(defn- response - [repo-bucket path] - (if-some [entries (seq (repo-indexing/get-path-entries repo-bucket path))] - (-> (repo-indexing/generate-index path entries) - (ring.response/response) - (ring.response/content-type "text/html;charset=utf-8")) - not-found-response)) - -(defn- validate-path - "Checks path to see if it is one we would even have in s3, returning a 404 w/o - calling s3 or caching if not. This is to reduce the work we do & disk space we - use for caching." - [path] - (when (and path (not (re-find maven/repo-path-regex path))) - [not-found-response 0])) - -(defn index-for-path - [{:keys [cache-path repo-bucket]} path] - (let [path (repo-indexing/normalize-path path) - [response age] (or (validate-path path) - (get-cached-response cache-path path) - (cache-response cache-path path (response repo-bucket path)))] - (ring.response/header response - "Cache-Control" (format "s-maxage=%s" (- max-age age))))) - -(defn repo-lister - [cache-path] - ;; :repo-bucket gets assoc'ed on to this by component - {:cache-path cache-path}) diff --git a/test/clojars/unit/web/repo_listing_test.clj b/test/clojars/unit/web/repo_listing_test.clj deleted file mode 100644 index 3326f32a..00000000 --- a/test/clojars/unit/web/repo_listing_test.clj +++ /dev/null @@ -1,40 +0,0 @@ -(ns clojars.unit.web.repo-listing-test - (:require - [clojars.s3 :as s3] - [clojars.test-helper :as help] - [clojars.web.repo-listing :as repo-listing] - [clojure.java.io :as io] - [clojure.test :refer [deftest is testing use-fixtures]] - [matcher-combinators.test])) - -(def ^:private tmp-cache (io/file help/tmp-dir "clojars" "repo-listing-cache")) - -(defn- with-tmp-cache - [f] - (help/delete-file-recursively tmp-cache) - (f)) - -(use-fixtures :each with-tmp-cache) - -(defn- mock-repo-lister - [] - {:cache-path (.getAbsolutePath tmp-cache) - :repo-bucket (s3/mock-s3-client)}) - -(deftest index-for-path-should-lookup-and-cache-valid-paths - (doseq [path [nil "a/" "abc/b/" "a_b/c-1/1.0.4+Foo/"]] - (testing (format "with path '%s'" path) - (let [lister (mock-repo-lister) - result (repo-listing/index-for-path lister path)] - (is (= [['-list-entries [path]]] (s3/get-mock-calls (:repo-bucket lister)))) - (is (.exists (repo-listing/cache-file (:cache-path lister) path))) - (is (match? {:status 404} result)))))) - -(deftest index-for-path-should-not-lookup-or-cache-invalid-paths - (doseq [path ["a/b/=/" "?foo=bar/" "abc.b/"]] - (testing (format "with path '%s'" path) - (let [lister (mock-repo-lister) - result (repo-listing/index-for-path lister path)] - (is (empty? (s3/get-mock-calls (:repo-bucket lister)))) - (is (not (.exists (repo-listing/cache-file (:cache-path lister) path)))) - (is (match? {:status 404} result))))))