From 6cd8ca6a694ecaf2d8a663c60641f247e41d348f Mon Sep 17 00:00:00 2001 From: Ayman Bagabas Date: Fri, 31 Mar 2023 13:14:29 -0400 Subject: [PATCH] feat(server): add pull mirror repos --- server/backend/file/file.go | 30 ++++++++++++++++--- server/backend/file/repo.go | 8 +++++ server/backend/repo.go | 14 ++++++++- server/cmd/create.go | 19 ++++++++++-- server/cron/cron.go | 58 +++++++++++++++++++++++++++++++++++++ server/jobs.go | 40 +++++++++++++++++++++++++ server/server.go | 12 ++++++++ server/ssh.go | 2 +- 8 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 server/cron/cron.go create mode 100644 server/jobs.go diff --git a/server/backend/file/file.go b/server/backend/file/file.go index cf0b368fe..230a83592 100644 --- a/server/backend/file/file.go +++ b/server/backend/file/file.go @@ -51,6 +51,7 @@ const ( private = "private" projectName = "project-name" settings = "settings" + mirror = "mirror" ) var ( @@ -591,12 +592,19 @@ func (fb *FileBackend) SetProjectName(repo string, name string) error { return os.WriteFile(filepath.Join(fb.reposPath(), repo, projectName), []byte(name), 0600) } +// IsMirror returns true if the given repo is a mirror. +func (fb *FileBackend) IsMirror(repo string) bool { + repo = utils.SanitizeRepo(repo) + ".git" + r := &Repo{path: filepath.Join(fb.reposPath(), repo), root: fb.reposPath()} + return r.IsMirror() +} + // CreateRepository creates a new repository. // // Created repositories are always bare. // // It implements backend.Backend. -func (fb *FileBackend) CreateRepository(repo string, private bool) (backend.Repository, error) { +func (fb *FileBackend) CreateRepository(repo string, opts backend.RepositoryOptions) (backend.Repository, error) { name := utils.SanitizeRepo(repo) repo = name + ".git" rp := filepath.Join(fb.reposPath(), repo) @@ -604,6 +612,20 @@ func (fb *FileBackend) CreateRepository(repo string, private bool) (backend.Repo return nil, os.ErrExist } + if opts.Mirror != "" { + if err := git.Clone(opts.Mirror, rp, git.CloneOptions{ + Mirror: true, + }); err != nil { + logger.Debug("failed to clone mirror repository", "err", err) + return nil, err + } + + if err := os.WriteFile(filepath.Join(rp, mirror), nil, 0600); err != nil { + logger.Debug("failed to create mirror file", "err", err) + return nil, err + } + } + rr, err := git.Init(rp, true) if err != nil { logger.Debug("failed to create repository", "err", err) @@ -615,17 +637,17 @@ func (fb *FileBackend) CreateRepository(repo string, private bool) (backend.Repo return nil, err } - if err := fb.SetPrivate(repo, private); err != nil { + if err := fb.SetPrivate(repo, opts.Private); err != nil { logger.Debug("failed to set private status", "err", err) return nil, err } - if err := fb.SetDescription(repo, ""); err != nil { + if err := fb.SetDescription(repo, opts.Description); err != nil { logger.Debug("failed to set description", "err", err) return nil, err } - if err := fb.SetProjectName(repo, name); err != nil { + if err := fb.SetProjectName(repo, opts.ProjectName); err != nil { logger.Debug("failed to set project name", "err", err) return nil, err } diff --git a/server/backend/file/repo.go b/server/backend/file/repo.go index 388820104..93d7dab9b 100644 --- a/server/backend/file/repo.go +++ b/server/backend/file/repo.go @@ -57,6 +57,14 @@ func (r *Repo) IsPrivate() bool { return err == nil } +// IsMirror returns whether the repository is a mirror. +// +// It implements backend.Repository. +func (r *Repo) IsMirror() bool { + _, err := os.Stat(filepath.Join(r.path, mirror)) + return err == nil +} + // Open returns the underlying git.Repository. // // It implements backend.Repository. diff --git a/server/backend/repo.go b/server/backend/repo.go index 9fcad1c45..b429a7a01 100644 --- a/server/backend/repo.go +++ b/server/backend/repo.go @@ -5,6 +5,14 @@ import ( "golang.org/x/crypto/ssh" ) +// RepositoryOptions are options for creating a new repository. +type RepositoryOptions struct { + Private bool + Mirror string + Description string + ProjectName string +} + // RepositoryStore is an interface for managing repositories. type RepositoryStore interface { // Repository finds the given repository. @@ -12,7 +20,7 @@ type RepositoryStore interface { // Repositories returns a list of all repositories. Repositories() ([]Repository, error) // CreateRepository creates a new repository. - CreateRepository(name string, private bool) (Repository, error) + CreateRepository(name string, opts RepositoryOptions) (Repository, error) // DeleteRepository deletes a repository. DeleteRepository(name string) error // RenameRepository renames a repository. @@ -33,6 +41,8 @@ type RepositoryMetadata interface { IsPrivate(repo string) bool // SetPrivate sets whether the repository is private. SetPrivate(repo string, private bool) error + // IsMirror returns whether the repository is a mirror. + IsMirror(repo string) bool } // RepositoryAccess is an interface for managing repository access. @@ -67,6 +77,8 @@ type Repository interface { Description() string // IsPrivate returns whether the repository is private. IsPrivate() bool + // IsMirror returns whether the repository is a mirror. + IsMirror() bool // Open returns the underlying git.Repository. Open() (*git.Repository, error) } diff --git a/server/cmd/create.go b/server/cmd/create.go index d7cfdb04d..891ad94f9 100644 --- a/server/cmd/create.go +++ b/server/cmd/create.go @@ -1,11 +1,17 @@ package cmd -import "github.com/spf13/cobra" +import ( + "github.com/charmbracelet/soft-serve/server/backend" + "github.com/spf13/cobra" +) // createCommand is the command for creating a new repository. func createCommand() *cobra.Command { var private bool var description string + var mirror string + var projectName string + cmd := &cobra.Command{ Use: "create REPOSITORY", Short: "Create a new repository", @@ -14,13 +20,22 @@ func createCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { cfg, _ := fromContext(cmd) name := args[0] - if _, err := cfg.Backend.CreateRepository(name, private); err != nil { + if _, err := cfg.Backend.CreateRepository(name, backend.RepositoryOptions{ + Private: private, + Mirror: mirror, + Description: description, + ProjectName: projectName, + }); err != nil { return err } return nil }, } + cmd.Flags().BoolVarP(&private, "private", "p", false, "make the repository private") cmd.Flags().StringVarP(&description, "description", "d", "", "set the repository description") + cmd.Flags().StringVarP(&mirror, "mirror", "m", "", "set the mirror repository") + cmd.Flags().StringVarP(&projectName, "name", "n", "", "set the project name") + return cmd } diff --git a/server/cron/cron.go b/server/cron/cron.go new file mode 100644 index 000000000..9f98d5a69 --- /dev/null +++ b/server/cron/cron.go @@ -0,0 +1,58 @@ +package cron + +import ( + "context" + "time" + + "github.com/charmbracelet/log" + "github.com/robfig/cron/v3" +) + +// CronScheduler is a cron-like job scheduler. +type CronScheduler struct { + *cron.Cron + logger cron.Logger +} + +// Entry is a cron job. +type Entry struct { + ID cron.EntryID + Desc string + Spec string +} + +// cronLogger is a wrapper around the logger to make it compatible with the +// cron logger. +type cronLogger struct { + logger *log.Logger +} + +// Info logs routine messages about cron's operation. +func (l cronLogger) Info(msg string, keysAndValues ...interface{}) { + l.logger.Info(msg, keysAndValues...) +} + +// Error logs an error condition. +func (l cronLogger) Error(err error, msg string, keysAndValues ...interface{}) { + l.logger.Error(msg, append(keysAndValues, "err", err)...) +} + +// NewCronScheduler returns a new Cron. +func NewCronScheduler() *CronScheduler { + logger := cronLogger{log.WithPrefix("server.cron")} + return &CronScheduler{ + Cron: cron.New(cron.WithLogger(logger)), + } +} + +// Shutdonw gracefully shuts down the CronServer. +func (s *CronScheduler) Shutdown() { + ctx, cancel := context.WithTimeout(s.Cron.Stop(), 30*time.Second) + defer func() { cancel() }() + <-ctx.Done() +} + +// Start starts the CronServer. +func (s *CronScheduler) Start() { + s.Cron.Start() +} diff --git a/server/jobs.go b/server/jobs.go new file mode 100644 index 000000000..d9621b8b0 --- /dev/null +++ b/server/jobs.go @@ -0,0 +1,40 @@ +package server + +import ( + "github.com/charmbracelet/soft-serve/git" + "github.com/charmbracelet/soft-serve/server/backend" +) + +var ( + jobSpecs = map[string]string{ + "mirror": "@every 10m", + } +) + +// mirrorJob runs the (pull) mirror job task. +func mirrorJob(b backend.Backend) func() { + logger := logger.WithPrefix("server.mirrorJob") + return func() { + repos, err := b.Repositories() + if err != nil { + logger.Error("error getting repositories", "err", err) + return + } + + for _, repo := range repos { + if repo.IsMirror() { + logger.Debug("updating mirror", "repo", repo.Name()) + r, err := repo.Open() + if err != nil { + logger.Error("error opening repository", "repo", repo.Name(), "err", err) + continue + } + + cmd := git.NewCommand("remote", "update", "--prune") + if _, err := cmd.RunInDir(r.Path); err != nil { + logger.Error("error running git remote update", "repo", repo.Name(), "err", err) + } + } + } + } +} diff --git a/server/server.go b/server/server.go index 976b1b9fa..bcb5d904d 100644 --- a/server/server.go +++ b/server/server.go @@ -11,6 +11,7 @@ import ( "github.com/charmbracelet/soft-serve/server/backend" "github.com/charmbracelet/soft-serve/server/backend/file" "github.com/charmbracelet/soft-serve/server/config" + "github.com/charmbracelet/soft-serve/server/cron" "github.com/charmbracelet/ssh" "golang.org/x/sync/errgroup" ) @@ -25,6 +26,7 @@ type Server struct { GitDaemon *GitDaemon HTTPServer *HTTPServer StatsServer *StatsServer + Cron *cron.CronScheduler Config *config.Config Backend backend.Backend } @@ -57,9 +59,14 @@ func NewServer(cfg *config.Config) (*Server, error) { } srv := &Server{ + Cron: cron.NewCronScheduler(), Config: cfg, Backend: cfg.Backend, } + + // Add cron jobs. + srv.Cron.AddFunc(jobSpecs["mirror"], mirrorJob(cfg.Backend)) + srv.SSHServer, err = NewSSHServer(cfg, srv) if err != nil { return nil, err @@ -114,6 +121,11 @@ func (s *Server) Start() error { } return nil }) + errg.Go(func() error { + log.Print("Starting cron scheduler") + s.Cron.Start() + return nil + }) return errg.Wait() } diff --git a/server/ssh.go b/server/ssh.go index 64f48a00f..06916813d 100644 --- a/server/ssh.go +++ b/server/ssh.go @@ -185,7 +185,7 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware { return } if _, err := cfg.Backend.Repository(name); err != nil { - if _, err := cfg.Backend.CreateRepository(name, false); err != nil { + if _, err := cfg.Backend.CreateRepository(name, backend.RepositoryOptions{Private: false}); err != nil { log.Errorf("failed to create repo: %s", err) sshFatal(s, err) return