Skip to content

Commit

Permalink
Implement repo dir listing
Browse files Browse the repository at this point in the history
This will generate a list of the repo given a path prefix. This is
intended to be used only via fastly for repo.clojars.org, so isn't
navigable if accessed directly.
  • Loading branch information
tobias committed May 11, 2023
1 parent c449eec commit cf72fee
Show file tree
Hide file tree
Showing 6 changed files with 180 additions and 51 deletions.
23 changes: 23 additions & 0 deletions src/clojars/routes/repo_listing.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
(ns clojars.routes.repo-listing
(:require
[clojars.web.repo-listing :as repo-listing]
[compojure.core :as compojure :refer [GET HEAD]]
[ring.util.response :as ring.response]))

(defn- repo-listing
[repo-bucket path]
(-> (repo-listing/index repo-bucket path)
(ring.response/response)
(ring.response/content-type "text/html;charset=utf-8")
;; Instruct fastly to cache this result for 15 minutes
(ring.response/header "Cache-Control" "s-maxage=900")))

(defn routes
[repo-bucket]
(compojure/routes
(GET ["/repo-listing"]
{{:keys [path]} :params}
(repo-listing repo-bucket path))
(HEAD ["/repo-listing"]
{{:keys [path]} :params}
(repo-listing repo-bucket path))))
2 changes: 1 addition & 1 deletion src/clojars/s3.clj
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@
k)
[k-segment & more] (str/split k-sans-prefix #"/")]
(if more
{:Prefix (format "%s%s/" prefix k-segment)}
{:Prefix (format "%s%s/" (or prefix "") k-segment)}
(mock-object-entry k (get @state k))))))
(distinct)))
(-list-objects [_ prefix]
Expand Down
2 changes: 1 addition & 1 deletion src/clojars/system.clj
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@
(component/system-using
{:app [:clojars-app]
:clojars-app [:db :github :gitlab :error-reporter :http-client
:mailer :stats :search :storage]
:mailer :repo-bucket :stats :search :storage]
:http [:app]
:notifications [:db :mailer]
:storage [:error-reporter :repo-bucket]}))))
101 changes: 52 additions & 49 deletions src/clojars/web.clj
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
[clojars.routes.artifact :as artifact]
[clojars.routes.group :as group]
[clojars.routes.repo :as repo]
[clojars.routes.repo-listing :as repo-listing]
[clojars.routes.session :as session]
[clojars.routes.token :as token]
[clojars.routes.token-breach :as token-breach]
Expand Down Expand Up @@ -46,51 +47,54 @@
:error-message "The page query parameter must be an integer."
:status 400})))))

(defn- main-routes [db stats search-obj mailer]
(routes
(GET "/" _
(try-account
#(if %
(dashboard db %)
(index-page db stats %))))
(GET "/search" {:keys [params]}
(try-account
#(let [validated-params (if (:page params)
(assoc params :page (try-parse-page (:page params)))
params)]
(search search-obj % validated-params))))
(GET "/projects" {:keys [params]}
(try-account
#(let [validated-params (if (:page params)
(assoc params :page (try-parse-page (:page params)))
params)]
(browse db % validated-params))))
(GET "/security" []
(try-account
#(html-doc "Security" {:account %}
(raw (slurp (io/resource "security.html"))))))
(GET "/dmca" []
(try-account
#(html-doc "DMCA" {:account %}
(raw (slurp (io/resource "dmca.html"))))))
session/routes
(group/routes db)
(artifact/routes db stats)
;; user routes must go after artifact routes
;; since they both catch /:identifier
(user/routes db mailer)
(verify/routes db)
(token/routes db)
(api/routes db stats)
(GET "/error" _ (throw (Exception. "What!? You really want an error?")))
(PUT "*" _ {:status 405 :headers {} :body "Did you mean to use /repo?"})
(ANY "*" _
(try-account
#(not-found
(html-doc "Page not found" {:account %}
[:div.small-section
[:h1 "Page not found"]
[:p "Thundering typhoons! I think we lost it. Sorry!"]]))))))
(defn- main-routes
[{:as _system :keys [db mailer repo-bucket search-obj stats]}]
(let [db (:spec db)]
(routes
(GET "/" _
(try-account
#(if %
(dashboard db %)
(index-page db stats %))))
(GET "/search" {:keys [params]}
(try-account
#(let [validated-params (if (:page params)
(assoc params :page (try-parse-page (:page params)))
params)]
(search search-obj % validated-params))))
(GET "/projects" {:keys [params]}
(try-account
#(let [validated-params (if (:page params)
(assoc params :page (try-parse-page (:page params)))
params)]
(browse db % validated-params))))
(GET "/security" []
(try-account
#(html-doc "Security" {:account %}
(raw (slurp (io/resource "security.html"))))))
(GET "/dmca" []
(try-account
#(html-doc "DMCA" {:account %}
(raw (slurp (io/resource "dmca.html"))))))
session/routes
(repo-listing/routes repo-bucket)
(group/routes db)
(artifact/routes db stats)
;; user routes must go after artifact routes
;; since they both catch /:identifier
(user/routes db mailer)
(verify/routes db)
(token/routes db)
(api/routes db stats)
(GET "/error" _ (throw (Exception. "What!? You really want an error?")))
(PUT "*" _ {:status 405 :headers {} :body "Did you mean to use /repo?"})
(ANY "*" _
(try-account
#(not-found
(html-doc "Page not found" {:account %}
[:div.small-section
[:h1 "Page not found"]
[:p "Thundering typhoons! I think we lost it. Sorry!"]])))))))

(def ^:private defaults-config
(-> ring-defaults/secure-site-defaults
Expand All @@ -102,14 +106,13 @@
(dissoc :session)))

(defn clojars-app
[{:keys [db
[{:as system
:keys [db
error-reporter
http-client
github
gitlab
mailer
search
stats
storage]}]
(let [db (:spec db)]
(routes
Expand All @@ -131,7 +134,7 @@
(-> (token-breach/routes db)
(wrap-exceptions error-reporter)
(log/wrap-request-context))
(-> (main-routes db stats search mailer)
(-> (main-routes system)
(friend/authenticate
{:credential-fn (auth/password-credential-fn db)
:workflows [(auth/interactive-form-with-mfa-workflow)
Expand Down
3 changes: 3 additions & 0 deletions src/clojars/web/common.clj
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,9 @@
(def format-timestamp
(partial format-date* (SimpleDateFormat. "MMM d, yyyy HH:mm:ss Z")))

(def format-date-with-time
(partial format-date* (SimpleDateFormat. "yyyy-MM-dd HH:mm")))

(defn page-nav [current-page total-pages & {:keys [base-path] :or {base-path "/projects?page="}}]
(let [previous-text (raw "← Previous")
next-text (raw "Next →")
Expand Down
100 changes: 100 additions & 0 deletions src/clojars/web/repo_listing.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
(ns clojars.web.repo-listing
(:require
[clojars.s3 :as s3]
[clojars.web.common :as common]
[clojars.web.safe-hiccup :as safe-hiccup]
[clojure.string :as str]
[hiccup.element :as el]))

(defn- sort-entries
[entries]
(sort
(fn [e1 e2]
(cond
(and (:Prefix e1) (:Prefix e2)) (compare (:Prefix e1) (:Prefix e2))
(:Prefix e1) -1
(:Prefix e2) 1
:else (compare (:Key e1) (:Key e2))))
entries))

(defn- last-segment
[path]
(peek (str/split path #"/")))

(defn- entry-line-dispatch
[entry]
(if (:Prefix entry)
:prefix
:file))

(defmulti entry-line
#'entry-line-dispatch)

(def ^:private name-col-width 50)
(def ^:private date-col-width 16)
(def ^:private size-col-width 10)

(defn- blanks
[max-len content-str]
(safe-hiccup/raw (apply str (repeat (- max-len (count content-str)) " "))))

(def ^:private dash "-")

(defmethod entry-line :prefix
[{:keys [Prefix]}]
(let [suffix (format "%s/" (last-segment Prefix))]
(list
(el/link-to {:title suffix} suffix suffix)
(blanks name-col-width suffix)
(blanks date-col-width dash)
dash
(blanks size-col-width dash)
dash
"\n")))

(defmethod entry-line :file
[{:keys [ContentLength Key LastModified]}]
(let [file-name (last-segment Key)
size (str ContentLength)]
(list
(el/link-to {:title file-name} file-name file-name)
(blanks name-col-width file-name)
(common/format-date-with-time LastModified)
(blanks size-col-width size)
size
"\n")))

(defn- normalize-path
[path]
(let [path (cond
(str/blank? path) nil
(= "/" path) nil
(str/starts-with? path "/") (subs path 1)
:else path)]
(if (and (some? path)
(not (str/ends-with? path "/")))
(str path "/")
path)))

(defn index
[s3 path]
(let [path (normalize-path path)
entries (sort-entries (s3/list-entries s3 path))]
(safe-hiccup/html5
{:lang "en"}
[:head
[:meta {:charset "utf-8"}]
[:meta {:name "viewport" :content "width=device-width,initial-scale=1"}]
[:title (format "Clojars Repository: %s" path)]]
[:body
[:header
[:h1 (or path "/")]]
[:hr]
[:main
[:pre#contents
(when (some? path)
(list
(el/link-to "../" "../")
"\n"))
(mapcat entry-line entries)]]
[:hr]])))

0 comments on commit cf72fee

Please sign in to comment.