diff --git a/cmd/soft/root.go b/cmd/soft/root.go index 273c9d6a3..f78c22606 100644 --- a/cmd/soft/root.go +++ b/cmd/soft/root.go @@ -54,6 +54,9 @@ func init() { func main() { logger := NewDefaultLogger() + // Set global logger + log.SetDefault(logger) + // Set the max number of processes to the number of CPUs // This is useful when running soft serve in a container if _, err := maxprocs.Set(maxprocs.Logger(logger.Debugf)); err != nil { diff --git a/git/repo.go b/git/repo.go index 67a0e286d..a3aa2f324 100644 --- a/git/repo.go +++ b/git/repo.go @@ -205,13 +205,6 @@ func (r *Repository) CommitsByPage(ref *Reference, page, size int) (Commits, err return commits, nil } -// UpdateServerInfo updates the repository server info. -func (r *Repository) UpdateServerInfo() error { - cmd := git.NewCommand("update-server-info") - _, err := cmd.RunInDir(r.Path) - return err -} - // Config returns the config value for the given key. func (r *Repository) Config(key string, opts ...ConfigOptions) (string, error) { dir, err := gitDir(r.Repository) diff --git a/git/server.go b/git/server.go new file mode 100644 index 000000000..e868b1a29 --- /dev/null +++ b/git/server.go @@ -0,0 +1,18 @@ +package git + +import ( + "context" + + "github.com/gogs/git-module" +) + +// UpdateServerInfo updates the server info file for the given repo path. +func UpdateServerInfo(ctx context.Context, path string) error { + if !isGitDir(path) { + return ErrNotAGitRepository + } + + cmd := git.NewCommand("update-server-info").WithContext(ctx).WithTimeout(-1) + _, err := cmd.RunInDir(path) + return err +} diff --git a/git/utils.go b/git/utils.go index 2b3d28728..3710e172d 100644 --- a/git/utils.go +++ b/git/utils.go @@ -1,6 +1,7 @@ package git import ( + "os" "path/filepath" "github.com/gobwas/glob" @@ -49,3 +50,25 @@ func LatestFile(repo *Repository, pattern string) (string, string, error) { } return "", "", ErrFileNotFound } + +// Returns true if path is a directory containing an `objects` directory and a +// `HEAD` file. +func isGitDir(path string) bool { + stat, err := os.Stat(filepath.Join(path, "objects")) + if err != nil { + return false + } + if !stat.IsDir() { + return false + } + + stat, err = os.Stat(filepath.Join(path, "HEAD")) + if err != nil { + return false + } + if stat.IsDir() { + return false + } + + return true +} diff --git a/internal/log/log.go b/internal/log/log.go index 7389b80fb..b6c4b1443 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -32,6 +32,10 @@ func NewDefaultLogger() *log.Logger { if debug, _ := strconv.ParseBool(os.Getenv("SOFT_SERVE_DEBUG")); debug { logger.SetLevel(log.DebugLevel) + + if verbose, _ := strconv.ParseBool(os.Getenv("SOFT_SERVE_VERBOSE")); verbose { + logger.SetReportCaller(true) + } } logger.SetTimeFormat(cfg.Log.TimeFormat) diff --git a/server/backend/sqlite/hooks.go b/server/backend/sqlite/hooks.go index ff39046f4..972b3f31d 100644 --- a/server/backend/sqlite/hooks.go +++ b/server/backend/sqlite/hooks.go @@ -36,16 +36,6 @@ func (d *SqliteBackend) PostUpdate(stdout io.Writer, stderr io.Writer, repo stri var wg sync.WaitGroup - // Update server info - wg.Add(1) - go func() { - defer wg.Done() - if err := updateServerInfo(d, repo); err != nil { - d.logger.Error("error updating server-info", "repo", repo, "err", err) - return - } - }() - // Populate last-modified file. wg.Add(1) go func() { @@ -59,20 +49,6 @@ func (d *SqliteBackend) PostUpdate(stdout io.Writer, stderr io.Writer, repo stri wg.Wait() } -func updateServerInfo(d *SqliteBackend, repo string) error { - rr, err := d.Repository(repo) - if err != nil { - return err - } - - r, err := rr.Open() - if err != nil { - return err - } - - return r.UpdateServerInfo() -} - func populateLastModified(d *SqliteBackend, repo string) error { var rr *Repo _rr, err := d.Repository(repo) diff --git a/server/backend/sqlite/sqlite.go b/server/backend/sqlite/sqlite.go index 91527e3fc..3273373ea 100644 --- a/server/backend/sqlite/sqlite.go +++ b/server/backend/sqlite/sqlite.go @@ -151,17 +151,12 @@ func (d *SqliteBackend) CreateRepository(name string, opts backend.RepositoryOpt return err } - rr, err := git.Init(rp, true) + _, err := git.Init(rp, true) if err != nil { d.logger.Debug("failed to create repository", "err", err) return err } - if err := rr.UpdateServerInfo(); err != nil { - d.logger.Debug("failed to update server info", "err", err) - return err - } - return nil }); err != nil { d.logger.Debug("failed to create repository in database", "err", err) diff --git a/server/git/git.go b/server/git/git.go index 6a772d36b..8f8ae3d7c 100644 --- a/server/git/git.go +++ b/server/git/git.go @@ -69,6 +69,8 @@ func EnsureWithin(reposDir string, repo string) error { return nil } +// EnsureDefaultBranch ensures the repo has a default branch. +// It will prefer choosing "main" or "master" if available. func EnsureDefaultBranch(ctx context.Context, scmd ServiceCommand) error { r, err := git.Open(scmd.Dir) if err != nil { diff --git a/server/git/service.go b/server/git/service.go index c4bfcc5d0..073001840 100644 --- a/server/git/service.go +++ b/server/git/service.go @@ -9,6 +9,7 @@ import ( "os/exec" "strings" + "github.com/charmbracelet/log" "golang.org/x/sync/errgroup" ) @@ -66,21 +67,35 @@ func gitServiceHandler(ctx context.Context, svc Service, scmd ServiceCommand) er scmd.CmdFunc(cmd) } - stdin, err := cmd.StdinPipe() - if err != nil { - return err + var ( + err error + stdin io.WriteCloser + stdout io.ReadCloser + stderr io.ReadCloser + ) + + if scmd.Stdin != nil { + stdin, err = cmd.StdinPipe() + if err != nil { + return err + } } - stdout, err := cmd.StdoutPipe() - if err != nil { - return err + if scmd.Stdout != nil { + stdout, err = cmd.StdoutPipe() + if err != nil { + return err + } } - stderr, err := cmd.StderrPipe() - if err != nil { - return err + if scmd.Stderr != nil { + stderr, err = cmd.StderrPipe() + if err != nil { + return err + } } + log.Debugf("git service command in %q: %s", cmd.Dir, cmd.String()) if err := cmd.Start(); err != nil { return err } @@ -88,36 +103,71 @@ func gitServiceHandler(ctx context.Context, svc Service, scmd ServiceCommand) er errg, ctx := errgroup.WithContext(ctx) // stdin - errg.Go(func() error { - defer stdin.Close() // nolint: errcheck - _, err := io.Copy(stdin, scmd.Stdin) - return err - }) + if scmd.Stdin != nil { + errg.Go(func() error { + if scmd.StdinHandler != nil { + return scmd.StdinHandler(scmd.Stdin, stdin) + } else { + return defaultStdinHandler(scmd.Stdin, stdin) + } + }) + } // stdout - errg.Go(func() error { - _, err := io.Copy(scmd.Stdout, stdout) - return err - }) + if scmd.Stdout != nil { + errg.Go(func() error { + if scmd.StdoutHandler != nil { + return scmd.StdoutHandler(scmd.Stdout, stdout) + } else { + return defaultStdoutHandler(scmd.Stdout, stdout) + } + }) + } // stderr - errg.Go(func() error { - _, err := io.Copy(scmd.Stderr, stderr) - return err - }) + if scmd.Stderr != nil { + errg.Go(func() error { + if scmd.StderrHandler != nil { + return scmd.StderrHandler(scmd.Stderr, stderr) + } else { + return defaultStderrHandler(scmd.Stderr, stderr) + } + }) + } return errors.Join(errg.Wait(), cmd.Wait()) } // ServiceCommand is used to run a git service command. type ServiceCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer - Dir string - Env []string - Args []string - CmdFunc func(*exec.Cmd) + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Dir string + Env []string + Args []string + + // Modifier functions + CmdFunc func(*exec.Cmd) + StdinHandler func(io.Reader, io.WriteCloser) error + StdoutHandler func(io.Writer, io.ReadCloser) error + StderrHandler func(io.Writer, io.ReadCloser) error +} + +func defaultStdinHandler(in io.Reader, stdin io.WriteCloser) error { + defer stdin.Close() // nolint: errcheck + _, err := io.Copy(stdin, in) + return err +} + +func defaultStdoutHandler(out io.Writer, stdout io.ReadCloser) error { + _, err := io.Copy(out, stdout) + return err +} + +func defaultStderrHandler(err io.Writer, stderr io.ReadCloser) error { + _, erro := io.Copy(err, stderr) + return erro } // UploadPack runs the git upload-pack protocol against the provided repo. diff --git a/server/web/git.go b/server/web/git.go new file mode 100644 index 000000000..2ca926591 --- /dev/null +++ b/server/web/git.go @@ -0,0 +1,459 @@ +package web + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/charmbracelet/log" + gitb "github.com/charmbracelet/soft-serve/git" + "github.com/charmbracelet/soft-serve/server/backend" + "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/git" + "github.com/charmbracelet/soft-serve/server/utils" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "goji.io/pat" + "goji.io/pattern" +) + +// GitRoute is a route for git services. +type GitRoute struct { + method string + pattern *regexp.Regexp + handler http.HandlerFunc + + cfg *config.Config + be backend.Backend + logger *log.Logger +} + +var _ Route = GitRoute{} + +// Match implements goji.Pattern. +func (g GitRoute) Match(r *http.Request) *http.Request { + if g.method != r.Method { + return nil + } + + re := g.pattern + ctx := r.Context() + if m := re.FindStringSubmatch(r.URL.Path); m != nil { + file := strings.Replace(r.URL.Path, m[1]+"/", "", 1) + repo := utils.SanitizeRepo(m[1]) + ".git" + + var service git.Service + switch { + case strings.HasSuffix(r.URL.Path, git.UploadPackService.String()): + service = git.UploadPackService + case strings.HasSuffix(r.URL.Path, git.ReceivePackService.String()): + service = git.ReceivePackService + } + + ctx = context.WithValue(ctx, pattern.Variable("service"), service.String()) + ctx = context.WithValue(ctx, pattern.Variable("dir"), filepath.Join(g.cfg.DataPath, "repos", repo)) + ctx = context.WithValue(ctx, pattern.Variable("repo"), repo) + ctx = context.WithValue(ctx, pattern.Variable("file"), file) + + if g.cfg != nil { + ctx = config.WithContext(ctx, g.cfg) + } + + if g.be != nil { + ctx = backend.WithContext(ctx, g.be.WithContext(ctx)) + } + + if g.logger != nil { + ctx = log.WithContext(ctx, g.logger) + } + + return r.WithContext(ctx) + } + + return nil +} + +// ServeHTTP implements http.Handler. +func (g GitRoute) ServeHTTP(w http.ResponseWriter, r *http.Request) { + g.handler(w, r) +} + +var ( + gitHttpReceiveCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "http", + Name: "git_receive_pack_total", + Help: "The total number of git push requests", + }, []string{"repo"}) + + gitHttpUploadCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "http", + Name: "git_upload_pack_total", + Help: "The total number of git fetch/pull requests", + }, []string{"repo", "file"}) +) + +func gitRoutes(ctx context.Context, logger *log.Logger) []Route { + routes := make([]Route, 0) + cfg := config.FromContext(ctx) + be := backend.FromContext(ctx) + + // Git services + // These routes don't handle authentication/authorization. + // This is handled through wrapping the handlers for each route. + // See below (withAccess). + // TODO: add lfs support + for _, route := range []GitRoute{ + { + pattern: regexp.MustCompile("(.*?)/git-upload-pack$"), + method: http.MethodPost, + handler: serviceRpc, + }, + { + pattern: regexp.MustCompile("(.*?)/git-receive-pack$"), + method: http.MethodPost, + handler: serviceRpc, + }, + { + pattern: regexp.MustCompile("(.*?)/info/refs$"), + method: http.MethodGet, + handler: getInfoRefs, + }, + { + pattern: regexp.MustCompile("(.*?)/HEAD$"), + method: http.MethodGet, + handler: getTextFile, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/info/alternates$"), + method: http.MethodGet, + handler: getTextFile, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/info/http-alternates$"), + method: http.MethodGet, + handler: getTextFile, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/info/packs$"), + method: http.MethodGet, + handler: getInfoPacks, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/info/[^/]*$"), + method: http.MethodGet, + handler: getTextFile, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/[0-9a-f]{2}/[0-9a-f]{38}$"), + method: http.MethodGet, + handler: getLooseObject, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/pack/pack-[0-9a-f]{40}\\.pack$"), + method: http.MethodGet, + handler: getPackFile, + }, + { + pattern: regexp.MustCompile("(.*?)/objects/pack/pack-[0-9a-f]{40}\\.idx$"), + method: http.MethodGet, + handler: getIdxFile, + }, + } { + route.cfg = cfg + route.be = be + route.logger = logger + route.handler = withAccess(route.handler) + routes = append(routes, route) + } + + return routes +} + +// withAccess handles auth. +func withAccess(fn http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + be := backend.FromContext(ctx) + logger := log.FromContext(ctx) + + if !be.AllowKeyless() { + renderForbidden(w) + return + } + + repo := pat.Param(r, "repo") + service := git.Service(pat.Param(r, "service")) + access := be.AccessLevel(repo, "") + + switch service { + case git.ReceivePackService: + if access < backend.ReadWriteAccess { + renderUnauthorized(w) + return + } + + // Create the repo if it doesn't exist. + if _, err := be.Repository(repo); err != nil { + if _, err := be.CreateRepository(repo, backend.RepositoryOptions{}); err != nil { + logger.Error("failed to create repository", "repo", repo, "err", err) + renderInternalServerError(w) + return + } + } + default: + if access < backend.ReadOnlyAccess { + renderUnauthorized(w) + return + } + } + + fn(w, r) + } +} + +func serviceRpc(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + logger := log.FromContext(ctx) + service, dir, repo := git.Service(pat.Param(r, "service")), pat.Param(r, "dir"), pat.Param(r, "repo") + + if !isSmart(r, service) { + renderForbidden(w) + return + } + + if service == git.ReceivePackService { + gitHttpReceiveCounter.WithLabelValues(repo) + } + + w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-result", service)) + w.Header().Set("Connection", "Keep-Alive") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + + version := r.Header.Get("Git-Protocol") + + cmd := git.ServiceCommand{ + Stdin: r.Body, + Stdout: w, + Dir: dir, + Args: []string{"--stateless-rpc"}, + } + + if len(version) != 0 { + cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_PROTOCOL=%s", version)) + } + + // Handle gzip encoding + cmd.StdinHandler = func(in io.Reader, stdin io.WriteCloser) (err error) { + // We know that `in` is an `io.ReadCloser` because it's `r.Body`. + reader := in.(io.ReadCloser) + defer reader.Close() // nolint: errcheck + switch r.Header.Get("Content-Encoding") { + case "gzip": + reader, err = gzip.NewReader(reader) + if err != nil { + return err + } + defer reader.Close() // nolint: errcheck + } + + _, err = io.Copy(stdin, reader) + return err + } + + // Handle buffered output + // Useful when using proxies + cmd.StdoutHandler = func(out io.Writer, stdout io.ReadCloser) error { + // We know that `out` is an `http.ResponseWriter`. + flusher, ok := out.(http.Flusher) + if !ok { + return fmt.Errorf("expected http.ResponseWriter to be an http.Flusher, got %T", out) + } + + p := make([]byte, 1024) + for { + nRead, err := stdout.Read(p) + if err == io.EOF { + break + } + nWrite, err := out.Write(p[:nRead]) + if err != nil { + return err + } + if nRead != nWrite { + return fmt.Errorf("failed to write data: %d read, %d written", nRead, nWrite) + } + flusher.Flush() + } + + return nil + } + + if err := service.Handler(ctx, cmd); err != nil { + logger.Errorf("error executing service: %s", err) + } +} + +func getInfoRefs(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + logger := log.FromContext(ctx) + dir, repo, file := pat.Param(r, "dir"), pat.Param(r, "repo"), pat.Param(r, "file") + service := getServiceType(r) + version := r.Header.Get("Git-Protocol") + + gitHttpUploadCounter.WithLabelValues(repo, file).Inc() + + if service != "" && (service == git.UploadPackService || service == git.ReceivePackService) { + // Smart HTTP + var refs bytes.Buffer + cmd := git.ServiceCommand{ + Stdout: &refs, + Dir: dir, + Args: []string{"--stateless-rpc", "--advertise-refs"}, + } + + if len(version) != 0 { + cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_PROTOCOL=%s", version)) + } + + if err := service.Handler(ctx, cmd); err != nil { + logger.Errorf("error executing service: %s", err) + renderNotFound(w) + return + } + + hdrNocache(w) + w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", service)) + w.WriteHeader(http.StatusOK) + if len(version) == 0 { + git.WritePktline(w, "# service="+service.String()) + } + + w.Write(refs.Bytes()) // nolint: errcheck + } else { + // Dumb HTTP + updateServerInfo(ctx, dir) // nolint: errcheck + hdrNocache(w) + sendFile("text/plain; charset=utf-8", w, r) + } +} + +func getInfoPacks(w http.ResponseWriter, r *http.Request) { + hdrCacheForever(w) + sendFile("text/plain; charset=utf-8", w, r) +} + +func getLooseObject(w http.ResponseWriter, r *http.Request) { + hdrCacheForever(w) + sendFile("application/x-git-loose-object", w, r) +} + +func getPackFile(w http.ResponseWriter, r *http.Request) { + hdrCacheForever(w) + sendFile("application/x-git-packed-objects", w, r) +} + +func getIdxFile(w http.ResponseWriter, r *http.Request) { + hdrCacheForever(w) + sendFile("application/x-git-packed-objects-toc", w, r) +} + +func getTextFile(w http.ResponseWriter, r *http.Request) { + hdrNocache(w) + sendFile("text/plain", w, r) +} + +func sendFile(contentType string, w http.ResponseWriter, r *http.Request) { + dir, file := pat.Param(r, "dir"), pat.Param(r, "file") + reqFile := filepath.Join(dir, file) + + f, err := os.Stat(reqFile) + if os.IsNotExist(err) { + renderNotFound(w) + return + } + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Size())) + w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat)) + http.ServeFile(w, r, reqFile) +} + +func getServiceType(r *http.Request) git.Service { + service := r.FormValue("service") + if !strings.HasPrefix(service, "git-") { + return "" + } + + return git.Service(service) +} + +func isSmart(r *http.Request, service git.Service) bool { + if r.Header.Get("Content-Type") == fmt.Sprintf("application/x-%s-request", service) { + return true + } + return false +} + +func updateServerInfo(ctx context.Context, dir string) error { + return gitb.UpdateServerInfo(ctx, dir) +} + +// HTTP error response handling functions + +func renderMethodNotAllowed(w http.ResponseWriter, r *http.Request) { + if r.Proto == "HTTP/1.1" { + w.WriteHeader(http.StatusMethodNotAllowed) + w.Write([]byte("Method Not Allowed")) // nolint: errcheck + } else { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) // nolint: errcheck + } +} + +func renderNotFound(w http.ResponseWriter) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not Found")) // nolint: errcheck +} + +func renderUnauthorized(w http.ResponseWriter) { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("Unauthorized")) // nolint: errcheck +} + +func renderForbidden(w http.ResponseWriter) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) // nolint: errcheck +} + +func renderInternalServerError(w http.ResponseWriter) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Internal Server Error")) // nolint: errcheck +} + +// Header writing functions + +func hdrNocache(w http.ResponseWriter) { + w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") +} + +func hdrCacheForever(w http.ResponseWriter) { + now := time.Now().Unix() + expires := now + 31536000 + w.Header().Set("Date", fmt.Sprintf("%d", now)) + w.Header().Set("Expires", fmt.Sprintf("%d", expires)) + w.Header().Set("Cache-Control", "public, max-age=31536000") +} diff --git a/server/web/goget.go b/server/web/goget.go new file mode 100644 index 000000000..7e7c8c9d6 --- /dev/null +++ b/server/web/goget.go @@ -0,0 +1,94 @@ +package web + +import ( + "net/http" + "net/url" + "path" + "text/template" + + "github.com/charmbracelet/soft-serve/server/backend" + "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/utils" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "goji.io/pattern" +) + +var goGetCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "soft_serve", + Subsystem: "http", + Name: "go_get_total", + Help: "The total number of go get requests", +}, []string{"repo"}) + +var repoIndexHTMLTpl = template.Must(template.New("index").Parse(` + + + + + + + +Redirecting to docs at godoc.org/{{ .ImportRoot }}/{{ .Repo }}... + +`)) + +// GoGetHandler handles go get requests. +type GoGetHandler struct { + cfg *config.Config + be backend.Backend +} + +var _ http.Handler = (*GoGetHandler)(nil) + +func (g GoGetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + repo := pattern.Path(r.Context()) + repo = utils.SanitizeRepo(repo) + be := g.be.WithContext(r.Context()) + + // Handle go get requests. + // + // Always return a 200 status code, even if the repo doesn't exist. + // + // https://golang.org/cmd/go/#hdr-Remote_import_paths + // https://go.dev/ref/mod#vcs-branch + if r.URL.Query().Get("go-get") == "1" { + repo := repo + importRoot, err := url.Parse(g.cfg.HTTP.PublicURL) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // find the repo + for { + if _, err := be.Repository(repo); err == nil { + break + } + + if repo == "" || repo == "." || repo == "/" { + return + } + + repo = path.Dir(repo) + } + + if err := repoIndexHTMLTpl.Execute(w, struct { + Repo string + Config *config.Config + ImportRoot string + }{ + Repo: url.PathEscape(repo), + Config: g.cfg, + ImportRoot: importRoot.Host, + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + goGetCounter.WithLabelValues(repo).Inc() + return + } + + http.NotFound(w, r) +} diff --git a/server/web/http.go b/server/web/http.go index b932f9416..7ff375cab 100644 --- a/server/web/http.go +++ b/server/web/http.go @@ -2,103 +2,31 @@ package web import ( "context" - "fmt" "net/http" - "net/url" - "path" - "path/filepath" - "regexp" - "strings" - "text/template" "time" - "github.com/charmbracelet/log" "github.com/charmbracelet/soft-serve/server/backend" "github.com/charmbracelet/soft-serve/server/config" - "github.com/charmbracelet/soft-serve/server/utils" - "github.com/dustin/go-humanize" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "goji.io" - "goji.io/pat" - "goji.io/pattern" ) -var ( - gitHttpCounter = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "soft_serve", - Subsystem: "http", - Name: "git_fetch_pull_total", - Help: "The total number of git fetch/pull requests", - }, []string{"repo", "file"}) - - goGetCounter = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "soft_serve", - Subsystem: "http", - Name: "go_get_total", - Help: "The total number of go get requests", - }, []string{"repo"}) -) - -// logWriter is a wrapper around http.ResponseWriter that allows us to capture -// the HTTP status code and bytes written to the response. -type logWriter struct { - http.ResponseWriter - code, bytes int -} - -func (r *logWriter) Write(p []byte) (int, error) { - written, err := r.ResponseWriter.Write(p) - r.bytes += written - return written, err -} - -// Note this is generally only called when sending an HTTP error, so it's -// important to set the `code` value to 200 as a default -func (r *logWriter) WriteHeader(code int) { - r.code = code - r.ResponseWriter.WriteHeader(code) -} - -func (s *HTTPServer) loggingMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - writer := &logWriter{code: http.StatusOK, ResponseWriter: w} - s.logger.Debug("request", - "method", r.Method, - "uri", r.RequestURI, - "addr", r.RemoteAddr) - next.ServeHTTP(writer, r) - elapsed := time.Since(start) - s.logger.Debug("response", - "status", fmt.Sprintf("%d %s", writer.code, http.StatusText(writer.code)), - "bytes", humanize.Bytes(uint64(writer.bytes)), - "time", elapsed) - }) -} - // HTTPServer is an http server. type HTTPServer struct { - ctx context.Context - cfg *config.Config - be backend.Backend - server *http.Server - dirHandler http.Handler - logger *log.Logger + ctx context.Context + cfg *config.Config + be backend.Backend + server *http.Server } +// NewHTTPServer creates a new HTTP server. func NewHTTPServer(ctx context.Context) (*HTTPServer, error) { cfg := config.FromContext(ctx) - mux := goji.NewMux() s := &HTTPServer{ - ctx: ctx, - cfg: cfg, - be: backend.FromContext(ctx), - logger: log.FromContext(ctx).WithPrefix("http"), - dirHandler: http.FileServer(http.Dir(filepath.Join(cfg.DataPath, "repos"))), + ctx: ctx, + cfg: cfg, + be: backend.FromContext(ctx), server: &http.Server{ Addr: cfg.HTTP.ListenAddr, - Handler: mux, + Handler: NewRouter(ctx), ReadHeaderTimeout: time.Second * 10, ReadTimeout: time.Second * 10, WriteTimeout: time.Second * 10, @@ -106,21 +34,6 @@ func NewHTTPServer(ctx context.Context) (*HTTPServer, error) { }, } - mux.Use(s.loggingMiddleware) - for _, m := range []Matcher{ - getInfoRefs, - getHead, - getAlternates, - getHTTPAlternates, - getInfoPacks, - getInfoFile, - getLooseObject, - getPackFile, - getIdxFile, - } { - mux.HandleFunc(NewPattern(m), s.handleGit) - } - mux.HandleFunc(pat.Get("/*"), s.handleIndex) return s, nil } @@ -141,193 +54,3 @@ func (s *HTTPServer) ListenAndServe() error { func (s *HTTPServer) Shutdown(ctx context.Context) error { return s.server.Shutdown(ctx) } - -// Pattern is a pattern for matching a URL. -// It matches against GET requests. -type Pattern struct { - match func(*url.URL) *match -} - -// NewPattern returns a new Pattern with the given matcher. -func NewPattern(m Matcher) *Pattern { - return &Pattern{ - match: m, - } -} - -// Match is a match for a URL. -// -// It implements goji.Pattern. -func (p *Pattern) Match(r *http.Request) *http.Request { - if r.Method != "GET" { - return nil - } - - if m := p.match(r.URL); m != nil { - ctx := context.WithValue(r.Context(), pattern.Variable("repo"), m.RepoPath) - ctx = context.WithValue(ctx, pattern.Variable("file"), m.FilePath) - return r.WithContext(ctx) - } - return nil -} - -// Matcher finds a match in a *url.URL. -type Matcher = func(*url.URL) *match - -var ( - getInfoRefs = func(u *url.URL) *match { - return matchSuffix(u.Path, "/info/refs") - } - - getHead = func(u *url.URL) *match { - return matchSuffix(u.Path, "/HEAD") - } - - getAlternates = func(u *url.URL) *match { - return matchSuffix(u.Path, "/objects/info/alternates") - } - - getHTTPAlternates = func(u *url.URL) *match { - return matchSuffix(u.Path, "/objects/info/http-alternates") - } - - getInfoPacks = func(u *url.URL) *match { - return matchSuffix(u.Path, "/objects/info/packs") - } - - getInfoFileRegexp = regexp.MustCompile(".*?(/objects/info/[^/]*)$") - getInfoFile = func(u *url.URL) *match { - return findStringSubmatch(u.Path, getInfoFileRegexp) - } - - getLooseObjectRegexp = regexp.MustCompile(".*?(/objects/[0-9a-f]{2}/[0-9a-f]{38})$") - getLooseObject = func(u *url.URL) *match { - return findStringSubmatch(u.Path, getLooseObjectRegexp) - } - - getPackFileRegexp = regexp.MustCompile(`.*?(/objects/pack/pack-[0-9a-f]{40}\.pack)$`) - getPackFile = func(u *url.URL) *match { - return findStringSubmatch(u.Path, getPackFileRegexp) - } - - getIdxFileRegexp = regexp.MustCompile(`.*?(/objects/pack/pack-[0-9a-f]{40}\.idx)$`) - getIdxFile = func(u *url.URL) *match { - return findStringSubmatch(u.Path, getIdxFileRegexp) - } -) - -// match represents a match for a URL. -type match struct { - RepoPath, FilePath string -} - -func matchSuffix(path, suffix string) *match { - if !strings.HasSuffix(path, suffix) { - return nil - } - repoPath := strings.Replace(path, suffix, "", 1) - filePath := strings.Replace(path, repoPath+"/", "", 1) - return &match{repoPath, filePath} -} - -func findStringSubmatch(path string, prefix *regexp.Regexp) *match { - m := prefix.FindStringSubmatch(path) - if m == nil { - return nil - } - suffix := m[1] - repoPath := strings.Replace(path, suffix, "", 1) - filePath := strings.Replace(path, repoPath+"/", "", 1) - return &match{repoPath, filePath} -} - -var repoIndexHTMLTpl = template.Must(template.New("index").Parse(` - - - - - - - -Redirecting to docs at godoc.org/{{ .ImportRoot }}/{{ .Repo }}... - -`)) - -func (s *HTTPServer) handleIndex(w http.ResponseWriter, r *http.Request) { - repo := pattern.Path(r.Context()) - repo = utils.SanitizeRepo(repo) - be := s.be.WithContext(r.Context()) - - // Handle go get requests. - // - // Always return a 200 status code, even if the repo doesn't exist. - // - // https://golang.org/cmd/go/#hdr-Remote_import_paths - // https://go.dev/ref/mod#vcs-branch - if r.URL.Query().Get("go-get") == "1" { - repo := repo - importRoot, err := url.Parse(s.cfg.HTTP.PublicURL) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // find the repo - for { - if _, err := be.Repository(repo); err == nil { - break - } - - if repo == "" || repo == "." || repo == "/" { - return - } - - repo = path.Dir(repo) - } - - if err := repoIndexHTMLTpl.Execute(w, struct { - Repo string - Config *config.Config - ImportRoot string - }{ - Repo: url.PathEscape(repo), - Config: s.cfg, - ImportRoot: importRoot.Host, - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - goGetCounter.WithLabelValues(repo).Inc() - return - } - - http.NotFound(w, r) -} - -func (s *HTTPServer) handleGit(w http.ResponseWriter, r *http.Request) { - repo := pat.Param(r, "repo") - repo = utils.SanitizeRepo(repo) + ".git" - be := s.be.WithContext(r.Context()) - if _, err := be.Repository(repo); err != nil { - s.logger.Debug("repository not found", "repo", repo, "err", err) - http.NotFound(w, r) - return - } - - if !s.cfg.Backend.AllowKeyless() { - http.Error(w, "Forbidden", http.StatusForbidden) - return - } - - access := s.cfg.Backend.AccessLevel(repo, "") - if access < backend.ReadOnlyAccess { - http.Error(w, "Unauthorized", http.StatusUnauthorized) - return - } - - file := pat.Param(r, "file") - gitHttpCounter.WithLabelValues(repo, file).Inc() - r.URL.Path = fmt.Sprintf("/%s/%s", repo, file) - s.dirHandler.ServeHTTP(w, r) -} diff --git a/server/web/logging.go b/server/web/logging.go new file mode 100644 index 000000000..f0f43a05c --- /dev/null +++ b/server/web/logging.go @@ -0,0 +1,84 @@ +package web + +import ( + "bufio" + "fmt" + "net" + "net/http" + "time" + + "github.com/charmbracelet/log" + "github.com/dustin/go-humanize" +) + +// logWriter is a wrapper around http.ResponseWriter that allows us to capture +// the HTTP status code and bytes written to the response. +type logWriter struct { + http.ResponseWriter + code, bytes int +} + +var _ http.ResponseWriter = (*logWriter)(nil) + +var _ http.Flusher = (*logWriter)(nil) + +var _ http.Hijacker = (*logWriter)(nil) + +var _ http.CloseNotifier = (*logWriter)(nil) + +// Write implements http.ResponseWriter. +func (r *logWriter) Write(p []byte) (int, error) { + written, err := r.ResponseWriter.Write(p) + r.bytes += written + return written, err +} + +// Note this is generally only called when sending an HTTP error, so it's +// important to set the `code` value to 200 as a default. +func (r *logWriter) WriteHeader(code int) { + r.code = code + r.ResponseWriter.WriteHeader(code) +} + +// Flush implements http.Flusher. +func (r *logWriter) Flush() { + if f, ok := r.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// CloseNotify implements http.CloseNotifier. +func (r *logWriter) CloseNotify() <-chan bool { + if cn, ok := r.ResponseWriter.(http.CloseNotifier); ok { + return cn.CloseNotify() + } + return nil +} + +// Hijack implements http.Hijacker. +func (r *logWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if h, ok := r.ResponseWriter.(http.Hijacker); ok { + return h.Hijack() + } + return nil, nil, fmt.Errorf("http.Hijacker not implemented") +} + +// NewLoggingMiddleware returns a new logging middleware. +func NewLoggingMiddleware(logger *log.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + writer := &logWriter{code: http.StatusOK, ResponseWriter: w} + logger.Debug("request", + "method", r.Method, + "uri", r.RequestURI, + "addr", r.RemoteAddr) + next.ServeHTTP(writer, r) + elapsed := time.Since(start) + logger.Debug("response", + "status", fmt.Sprintf("%d %s", writer.code, http.StatusText(writer.code)), + "bytes", humanize.Bytes(uint64(writer.bytes)), + "time", elapsed) + }) + } +} diff --git a/server/web/server.go b/server/web/server.go new file mode 100644 index 000000000..ea15e778f --- /dev/null +++ b/server/web/server.go @@ -0,0 +1,40 @@ +// Package server is the reusable server +package web + +import ( + "context" + "net/http" + + "github.com/charmbracelet/log" + "github.com/charmbracelet/soft-serve/server/backend" + "github.com/charmbracelet/soft-serve/server/config" + "goji.io" + "goji.io/pat" +) + +// Route is an interface for a route. +type Route interface { + http.Handler + goji.Pattern +} + +// NewRouter returns a new HTTP router. +func NewRouter(ctx context.Context) *goji.Mux { + mux := goji.NewMux() + cfg := config.FromContext(ctx) + be := backend.FromContext(ctx) + logger := log.FromContext(ctx).WithPrefix("http") + + // Middlewares + mux.Use(NewLoggingMiddleware(logger)) + + // Git routes + for _, service := range gitRoutes(ctx, logger) { + mux.Handle(service, service) + } + + // go-get handler + mux.Handle(pat.Get("/*"), GoGetHandler{cfg, be}) + + return mux +}