Skip to content

Commit

Permalink
Smart HTTP Git transport & partial clones (#291) (#332)
Browse files Browse the repository at this point in the history
* refactor: tidy up server git

use git services to implement handling git server commands
pass config to git as environment variables

* feat(git): enable partial clones

* feat(server): use smart http git backend

This implements the smart http git protocol which also supports
git-receive-pack service.
  • Loading branch information
aymanbagabas authored Jul 5, 2023
1 parent b60a7f2 commit e78725f
Show file tree
Hide file tree
Showing 18 changed files with 1,189 additions and 560 deletions.
3 changes: 3 additions & 0 deletions cmd/soft/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ func init() {
func main() {
logger := NewDefaultLogger()

// Set global logger
log.SetDefault(logger)

// Set the max number of processes to the number of CPUs
// This is useful when running soft serve in a container
if _, err := maxprocs.Set(maxprocs.Logger(logger.Debugf)); err != nil {
Expand Down
7 changes: 0 additions & 7 deletions git/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,13 +200,6 @@ func (r *Repository) CommitsByPage(ref *Reference, page, size int) (Commits, err
return commits, nil
}

// UpdateServerInfo updates the repository server info.
func (r *Repository) UpdateServerInfo() error {
cmd := git.NewCommand("update-server-info")
_, err := cmd.RunInDir(r.Path)
return err
}

// Config returns the config value for the given key.
func (r *Repository) Config(key string, opts ...ConfigOptions) (string, error) {
dir, err := gitDir(r.Repository)
Expand Down
18 changes: 18 additions & 0 deletions git/server.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package git

import (
"context"

"github.com/gogs/git-module"
)

// UpdateServerInfo updates the server info file for the given repo path.
func UpdateServerInfo(ctx context.Context, path string) error {
if !isGitDir(path) {
return ErrNotAGitRepository
}

cmd := git.NewCommand("update-server-info").WithContext(ctx).WithTimeout(-1)
_, err := cmd.RunInDir(path)
return err
}
23 changes: 23 additions & 0 deletions git/utils.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package git

import (
"os"
"path/filepath"

"github.com/gobwas/glob"
Expand Down Expand Up @@ -49,3 +50,25 @@ func LatestFile(repo *Repository, pattern string) (string, string, error) {
}
return "", "", ErrFileNotFound
}

// Returns true if path is a directory containing an `objects` directory and a
// `HEAD` file.
func isGitDir(path string) bool {
stat, err := os.Stat(filepath.Join(path, "objects"))
if err != nil {
return false
}
if !stat.IsDir() {
return false
}

stat, err = os.Stat(filepath.Join(path, "HEAD"))
if err != nil {
return false
}
if stat.IsDir() {
return false
}

return true
}
4 changes: 4 additions & 0 deletions internal/log/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ func NewDefaultLogger() *log.Logger {

if debug, _ := strconv.ParseBool(os.Getenv("SOFT_SERVE_DEBUG")); debug {
logger.SetLevel(log.DebugLevel)

if verbose, _ := strconv.ParseBool(os.Getenv("SOFT_SERVE_VERBOSE")); verbose {
logger.SetReportCaller(true)
}
}

logger.SetTimeFormat(cfg.Log.TimeFormat)
Expand Down
24 changes: 0 additions & 24 deletions server/backend/sqlite/hooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,6 @@ func (d *SqliteBackend) PostUpdate(stdout io.Writer, stderr io.Writer, repo stri

var wg sync.WaitGroup

// Update server info
wg.Add(1)
go func() {
defer wg.Done()
if err := updateServerInfo(d, repo); err != nil {
d.logger.Error("error updating server-info", "repo", repo, "err", err)
return
}
}()

// Populate last-modified file.
wg.Add(1)
go func() {
Expand All @@ -59,20 +49,6 @@ func (d *SqliteBackend) PostUpdate(stdout io.Writer, stderr io.Writer, repo stri
wg.Wait()
}

func updateServerInfo(d *SqliteBackend, repo string) error {
rr, err := d.Repository(repo)
if err != nil {
return err
}

r, err := rr.Open()
if err != nil {
return err
}

return r.UpdateServerInfo()
}

func populateLastModified(d *SqliteBackend, repo string) error {
var rr *Repo
_rr, err := d.Repository(repo)
Expand Down
7 changes: 1 addition & 6 deletions server/backend/sqlite/sqlite.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,17 +151,12 @@ func (d *SqliteBackend) CreateRepository(name string, opts backend.RepositoryOpt
return err
}

rr, err := git.Init(rp, true)
_, err := git.Init(rp, true)
if err != nil {
d.logger.Debug("failed to create repository", "err", err)
return err
}

if err := rr.UpdateServerInfo(); err != nil {
d.logger.Debug("failed to update server info", "err", err)
return err
}

return nil
}); err != nil {
d.logger.Debug("failed to create repository in database", "err", err)
Expand Down
34 changes: 34 additions & 0 deletions server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,40 @@ type Config struct {
Backend backend.Backend `yaml:"-"`
}

// Environ returns the config as a list of environment variables.
func (c *Config) Environ() []string {
envs := []string{}
if c == nil {
return envs
}

// TODO: do this dynamically
envs = append(envs, []string{
fmt.Sprintf("SOFT_SERVE_NAME=%s", c.Name),
fmt.Sprintf("SOFT_SERVE_DATA_PATH=%s", c.DataPath),
fmt.Sprintf("SOFT_SERVE_INITIAL_ADMIN_KEYS=%s", strings.Join(c.InitialAdminKeys, "\n")),
fmt.Sprintf("SOFT_SERVE_SSH_LISTEN_ADDR=%s", c.SSH.ListenAddr),
fmt.Sprintf("SOFT_SERVE_SSH_PUBLIC_URL=%s", c.SSH.PublicURL),
fmt.Sprintf("SOFT_SERVE_SSH_KEY_PATH=%s", c.SSH.KeyPath),
fmt.Sprintf("SOFT_SERVE_SSH_CLIENT_KEY_PATH=%s", c.SSH.ClientKeyPath),
fmt.Sprintf("SOFT_SERVE_SSH_MAX_TIMEOUT=%d", c.SSH.MaxTimeout),
fmt.Sprintf("SOFT_SERVE_SSH_IDLE_TIMEOUT=%d", c.SSH.IdleTimeout),
fmt.Sprintf("SOFT_SERVE_GIT_LISTEN_ADDR=%s", c.Git.ListenAddr),
fmt.Sprintf("SOFT_SERVE_GIT_MAX_TIMEOUT=%d", c.Git.MaxTimeout),
fmt.Sprintf("SOFT_SERVE_GIT_IDLE_TIMEOUT=%d", c.Git.IdleTimeout),
fmt.Sprintf("SOFT_SERVE_GIT_MAX_CONNECTIONS=%d", c.Git.MaxConnections),
fmt.Sprintf("SOFT_SERVE_HTTP_LISTEN_ADDR=%s", c.HTTP.ListenAddr),
fmt.Sprintf("SOFT_SERVE_HTTP_TLS_KEY_PATH=%s", c.HTTP.TLSKeyPath),
fmt.Sprintf("SOFT_SERVE_HTTP_TLS_CERT_PATH=%s", c.HTTP.TLSCertPath),
fmt.Sprintf("SOFT_SERVE_HTTP_PUBLIC_URL=%s", c.HTTP.PublicURL),
fmt.Sprintf("SOFT_SERVE_STATS_LISTEN_ADDR=%s", c.Stats.ListenAddr),
fmt.Sprintf("SOFT_SERVE_LOG_FORMAT=%s", c.Log.Format),
fmt.Sprintf("SOFT_SERVE_LOG_TIME_FORMAT=%s", c.Log.TimeFormat),
}...)

return envs
}

func parseConfig(path string) (*Config, error) {
dataPath := filepath.Dir(path)
cfg := &Config{
Expand Down
105 changes: 105 additions & 0 deletions server/daemon/conn.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package daemon

import (
"context"
"errors"
"net"
"sync"
"time"
)

// connections is a synchronizes access to to a net.Conn pool.
type connections struct {
m map[net.Conn]struct{}
mu sync.Mutex
}

func (m *connections) Add(c net.Conn) {
m.mu.Lock()
defer m.mu.Unlock()
m.m[c] = struct{}{}
}

func (m *connections) Close(c net.Conn) error {
m.mu.Lock()
defer m.mu.Unlock()
err := c.Close()
delete(m.m, c)
return err
}

func (m *connections) Size() int {
m.mu.Lock()
defer m.mu.Unlock()
return len(m.m)
}

func (m *connections) CloseAll() error {
m.mu.Lock()
defer m.mu.Unlock()
var err error
for c := range m.m {
err = errors.Join(err, c.Close())
delete(m.m, c)
}

return err
}

// serverConn is a wrapper around a net.Conn that closes the connection when
// the one of the timeouts is reached.
type serverConn struct {
net.Conn

initTimeout time.Duration
idleTimeout time.Duration
maxDeadline time.Time
closeCanceler context.CancelFunc
}

var _ net.Conn = (*serverConn)(nil)

func (c *serverConn) Write(p []byte) (n int, err error) {
c.updateDeadline()
n, err = c.Conn.Write(p)
if _, isNetErr := err.(net.Error); isNetErr && c.closeCanceler != nil {
c.closeCanceler()
}
return
}

func (c *serverConn) Read(b []byte) (n int, err error) {
c.updateDeadline()
n, err = c.Conn.Read(b)
if _, isNetErr := err.(net.Error); isNetErr && c.closeCanceler != nil {
c.closeCanceler()
}
return
}

func (c *serverConn) Close() (err error) {
err = c.Conn.Close()
if c.closeCanceler != nil {
c.closeCanceler()
}
return
}

func (c *serverConn) updateDeadline() {
switch {
case c.initTimeout > 0:
initTimeout := time.Now().Add(c.initTimeout)
c.initTimeout = 0
if initTimeout.Unix() < c.maxDeadline.Unix() || c.maxDeadline.IsZero() {
c.Conn.SetDeadline(initTimeout)
return
}
case c.idleTimeout > 0:
idleDeadline := time.Now().Add(c.idleTimeout)
if idleDeadline.Unix() < c.maxDeadline.Unix() || c.maxDeadline.IsZero() {
c.Conn.SetDeadline(idleDeadline)
return
}
}
c.Conn.SetDeadline(c.maxDeadline)
}
Loading

0 comments on commit e78725f

Please sign in to comment.