From b6e869c6c660b8c5544663ae78eb752054a1e473 Mon Sep 17 00:00:00 2001 From: nxyt Date: Fri, 23 Jun 2023 02:49:31 +0200 Subject: [PATCH] result caching groundwork --- cmd/root.go | 2 +- cmd/test.go | 7 ++- config/command.go | 10 ++- config/config_test.go | 2 +- config/optimus.yaml | 1 + config/service.go | 15 +++++ dirhash/dirhash.go | 117 ++++++++++++++++++++++++++++++++++++ dirhash/dirhash_test.go | 15 +++++ dirhash/read_ignore_test.go | 17 ++++++ flake.nix | 2 +- go.mod | 11 +++- go.sum | 11 ++++ lockfile/lockfile.go | 24 ++++++++ optimus.yaml | 13 +--- 14 files changed, 230 insertions(+), 17 deletions(-) create mode 100644 dirhash/dirhash.go create mode 100644 dirhash/dirhash_test.go create mode 100644 dirhash/read_ignore_test.go create mode 100644 lockfile/lockfile.go diff --git a/cmd/root.go b/cmd/root.go index 178fe39..861be4d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ var AppConfig config.Config var rootCmd = &cobra.Command{ Use: "optimus", Short: "Opinionated monorepo management framework", - Long: `Optimus is opinionated but extensible monorepo framework that can work with most web app workflows: standalone services, docker-compose projects and kubernetes clusters. It's easy to extend optimus to do most monorepo tasks using your favourite shell language.`, + Long: `Optimus is extensible monorepo framework that can work with most web app workflows: standalone services, docker-compose projects and kubernetes clusters. It's easy to extend optimus to do most monorepo tasks using your favourite shell language. Optimus supports caching of command results, so if your project didn't change then we won't rerun the command (unless you want to).`, } func Execute() { diff --git a/cmd/test.go b/cmd/test.go index 5f12df5..f4c0fd8 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -5,9 +5,12 @@ package cmd import ( "fmt" + "log" + "os" "sync" "github.com/spf13/cobra" + // "google.golang.org/appengine/log" ) // testCmd represents the test command @@ -47,7 +50,9 @@ to quickly create a Cobra application.`, for _, err := range errors { fmt.Println(err) } - panic("Not all tests passed") + + log.Print("Not all tests passed, exiting with code 1") + os.Exit(1) } }, } diff --git a/config/command.go b/config/command.go index 74f9676..c639a59 100644 --- a/config/command.go +++ b/config/command.go @@ -97,13 +97,19 @@ func ParseCmd(name string, root string, a any) Cmd { } func (c *Cmd) ToCobraCommand() cobra.Command { - return cobra.Command{ + cobraCmd := cobra.Command{ Use: c.Name, Short: c.Description, Run: func(cmd *cobra.Command, args []string) { if c.CommandFunc == nil { c.CommandFunc = c.GenerateCommandFunc() } + + force := cmd.Flag("force").Value.String() == "true" + if !force { + // check checksum from lockfile and if it's the same as now then skip test + } + err := c.CommandFunc() if err != nil { fmt.Println("Command failed") @@ -112,6 +118,8 @@ func (c *Cmd) ToCobraCommand() cobra.Command { } }, } + cobraCmd.Flags().BoolP("force", "f", false, "usage string") + return cobraCmd } func (c *Cmd) GenerateCommandFunc() func() error { diff --git a/config/config_test.go b/config/config_test.go index c2d5c9f..2347edb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -16,5 +16,5 @@ func TestLoadConfig(t *testing.T) { } func TestConfigCanBeMarchaled(t *testing.T) { - + // t.Fail() } diff --git a/config/optimus.yaml b/config/optimus.yaml index 46eee7d..4a656b6 100644 --- a/config/optimus.yaml +++ b/config/optimus.yaml @@ -7,6 +7,7 @@ services: echo 2 test: | echo "also testing" + false another_command: | echo "Just some other command" diff --git a/config/service.go b/config/service.go index 6a38b3e..c48d5e5 100644 --- a/config/service.go +++ b/config/service.go @@ -1,6 +1,11 @@ package config import ( + "optimus/dirhash" + "optimus/utils" + + "strings" + "github.com/spf13/cobra" ) @@ -20,6 +25,16 @@ type Service struct { DirHash string } +func (s *Service) UpdateDirhash() { + path := strings.Replace(s.Root, "./", "/", 1) + path = utils.ProjectRoot() + path + hash, err := dirhash.HashDir(path, make([]string, 0)) + if err != nil { + panic(err) + } + s.DirHash = hash +} + func ParseService(name string, a any) Service { // fmt.Println("Parsing service") s := Service{ diff --git a/dirhash/dirhash.go b/dirhash/dirhash.go new file mode 100644 index 0000000..425114c --- /dev/null +++ b/dirhash/dirhash.go @@ -0,0 +1,117 @@ +package dirhash + +import ( + "bufio" + "crypto" + "crypto/md5" + "io" + "io/fs" + "log" + "os" + "sort" +) + +func HashDir(path string, ignore []string) (string, error) { + files, err := os.ReadDir(path) + if err != nil { + return "", err + } + + if _, err := os.Stat(path + "/.gitignore"); !os.IsNotExist(err) { + ReadIgnore(path+"/.gitignore", &ignore) + } + + filtered := make([]fs.DirEntry, 0) + for _, de := range files { + deName := de.Name() + isIgnored := false + for _, v := range ignore { + if deName == v { + isIgnored = true + } + } + + if !isIgnored { + filtered = append(filtered, de) + } + } + files = filtered + + sort.SliceStable(files, func(i, j int) bool { + return files[i].Name() > files[j].Name() + }) + + hashArray := make([]string, 0) + + for _, de := range files { + p := path + string(os.PathSeparator) + de.Name() + if de.IsDir() { + newIgnore := make([]string, 0) + copy(newIgnore, ignore) + hash, err := HashDir(p, newIgnore) + if err != nil { + panic(err) + } + hashArray = append(hashArray, hash) + } else { + hash, err := HashFile(p) + if err != nil { + panic(err) + } + hashArray = append(hashArray, hash) + } + } + + hash := crypto.SHA256.New() + hash.Write([]byte(path)) + for _, v := range hashArray { + _, err := hash.Write([]byte(v)) + if err != nil { + panic(err) + } + } + res := string(hash.Sum(nil)) + + // return it + + return res, nil +} + +func HashFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + hash := md5.New() + _, err = io.Copy(hash, file) + if err != nil { + return "", err + } + sum := string(hash.Sum(nil)) + + return sum, nil +} + +func ReadIgnore(path string, ignore *[]string) error { + if ignore == nil { + panic("ignore slice is empty") + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + *ignore = append(*ignore, scanner.Text()) + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + + return nil +} diff --git a/dirhash/dirhash_test.go b/dirhash/dirhash_test.go new file mode 100644 index 0000000..da7955d --- /dev/null +++ b/dirhash/dirhash_test.go @@ -0,0 +1,15 @@ +package dirhash + +import ( + "fmt" + "optimus/utils" + "testing" +) + +func TestDirHash(t *testing.T) { + s, err := HashDir(utils.ProjectRoot(), make([]string, 0)) + if err != nil { + t.Error(err) + } + fmt.Println(s) +} diff --git a/dirhash/read_ignore_test.go b/dirhash/read_ignore_test.go new file mode 100644 index 0000000..ee236e7 --- /dev/null +++ b/dirhash/read_ignore_test.go @@ -0,0 +1,17 @@ +package dirhash + +import ( + "fmt" + "optimus/utils" + "testing" +) + +func TestReadIgnore(t *testing.T) { + ignores := make([]string, 0) + err := ReadIgnore(utils.ProjectRoot()+"/.gitignore", &ignores) + if err != nil { + t.Error(err) + } + + fmt.Println(ignores) +} diff --git a/flake.nix b/flake.nix index 68fbcca..bbcd49d 100644 --- a/flake.nix +++ b/flake.nix @@ -25,7 +25,7 @@ src = ./.; - vendorHash = "sha256-3tO/+Mnvl/wpS7Ro3XDIVrlYTGVM680mcC15/7ON6qM="; + vendorHash = "sha256-qWB4wz4JfxEh4LiixD5JK8/mmF/kmEKwTLh4mqEdDbA="; # vendorHash = pkgs.lib.fakeHash; meta = with pkgs.lib; { diff --git a/go.mod b/go.mod index fcf12a6..b76bf12 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,15 @@ module optimus go 1.20 +require ( + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.16.0 + google.golang.org/appengine v1.6.7 +) + require ( github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -11,13 +18,13 @@ require ( github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.16.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a53f2be..683e8c4 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -96,6 +99,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -247,6 +251,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -394,6 +400,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -457,6 +464,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/lockfile/lockfile.go b/lockfile/lockfile.go new file mode 100644 index 0000000..8a74f2f --- /dev/null +++ b/lockfile/lockfile.go @@ -0,0 +1,24 @@ +package lockfile + +import "optimus/utils" + +type Lockfile struct { + ProjectHash string + Services []ServiceTestLock +} + +type ServiceTestLock struct { + Name string + TestResult bool + Hash string + DependsOn []struct { + Name string + Hash string + } +} + +func LoadLockfile() Lockfile { + _ = utils.ProjectRoot() + + return Lockfile{} +} diff --git a/optimus.yaml b/optimus.yaml index 41085db..a2dc984 100644 --- a/optimus.yaml +++ b/optimus.yaml @@ -21,17 +21,10 @@ e2e_tests: # file: ./e2e_tests.nu services: - frontend: - root: ./config - dev: - shell: nu -c - run: | - ls + optimus: + root: . test: | - echo "testing" - # false - build: | - ls + go test ./... lscmd: root: ./config