Skip to content

Commit

Permalink
result caching groundwork
Browse files Browse the repository at this point in the history
  • Loading branch information
nxy7 committed Jun 23, 2023
1 parent c811f0f commit b6e869c
Show file tree
Hide file tree
Showing 14 changed files with 230 additions and 17 deletions.
2 changes: 1 addition & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ var AppConfig config.Config
var rootCmd = &cobra.Command{
Use: "optimus",
Short: "Opinionated monorepo management framework",
Long: `Optimus is opinionated but extensible monorepo framework that can work with most web app workflows: standalone services, docker-compose projects and kubernetes clusters. It's easy to extend optimus to do most monorepo tasks using your favourite shell language.`,
Long: `Optimus is extensible monorepo framework that can work with most web app workflows: standalone services, docker-compose projects and kubernetes clusters. It's easy to extend optimus to do most monorepo tasks using your favourite shell language. Optimus supports caching of command results, so if your project didn't change then we won't rerun the command (unless you want to).`,
}

func Execute() {
Expand Down
7 changes: 6 additions & 1 deletion cmd/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ package cmd

import (
"fmt"
"log"
"os"
"sync"

"github.com/spf13/cobra"
// "google.golang.org/appengine/log"
)

// testCmd represents the test command
Expand Down Expand Up @@ -47,7 +50,9 @@ to quickly create a Cobra application.`,
for _, err := range errors {
fmt.Println(err)
}
panic("Not all tests passed")

log.Print("Not all tests passed, exiting with code 1")
os.Exit(1)
}
},
}
Expand Down
10 changes: 9 additions & 1 deletion config/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,19 @@ func ParseCmd(name string, root string, a any) Cmd {
}

func (c *Cmd) ToCobraCommand() cobra.Command {
return cobra.Command{
cobraCmd := cobra.Command{
Use: c.Name,
Short: c.Description,
Run: func(cmd *cobra.Command, args []string) {
if c.CommandFunc == nil {
c.CommandFunc = c.GenerateCommandFunc()
}

force := cmd.Flag("force").Value.String() == "true"
if !force {
// check checksum from lockfile and if it's the same as now then skip test
}

err := c.CommandFunc()
if err != nil {
fmt.Println("Command failed")
Expand All @@ -112,6 +118,8 @@ func (c *Cmd) ToCobraCommand() cobra.Command {
}
},
}
cobraCmd.Flags().BoolP("force", "f", false, "usage string")
return cobraCmd
}

func (c *Cmd) GenerateCommandFunc() func() error {
Expand Down
2 changes: 1 addition & 1 deletion config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ func TestLoadConfig(t *testing.T) {
}

func TestConfigCanBeMarchaled(t *testing.T) {

// t.Fail()
}
1 change: 1 addition & 0 deletions config/optimus.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ services:
echo 2
test: |
echo "also testing"
false
another_command: |
echo "Just some other command"
Expand Down
15 changes: 15 additions & 0 deletions config/service.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
package config

import (
"optimus/dirhash"
"optimus/utils"

"strings"

"github.com/spf13/cobra"
)

Expand All @@ -20,6 +25,16 @@ type Service struct {
DirHash string
}

func (s *Service) UpdateDirhash() {
path := strings.Replace(s.Root, "./", "/", 1)
path = utils.ProjectRoot() + path
hash, err := dirhash.HashDir(path, make([]string, 0))
if err != nil {
panic(err)
}
s.DirHash = hash
}

func ParseService(name string, a any) Service {
// fmt.Println("Parsing service")
s := Service{
Expand Down
117 changes: 117 additions & 0 deletions dirhash/dirhash.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package dirhash

import (
"bufio"
"crypto"
"crypto/md5"
"io"
"io/fs"
"log"
"os"
"sort"
)

func HashDir(path string, ignore []string) (string, error) {
files, err := os.ReadDir(path)
if err != nil {
return "", err
}

if _, err := os.Stat(path + "/.gitignore"); !os.IsNotExist(err) {
ReadIgnore(path+"/.gitignore", &ignore)
}

filtered := make([]fs.DirEntry, 0)
for _, de := range files {
deName := de.Name()
isIgnored := false
for _, v := range ignore {
if deName == v {
isIgnored = true
}
}

if !isIgnored {
filtered = append(filtered, de)
}
}
files = filtered

sort.SliceStable(files, func(i, j int) bool {
return files[i].Name() > files[j].Name()
})

hashArray := make([]string, 0)

for _, de := range files {
p := path + string(os.PathSeparator) + de.Name()
if de.IsDir() {
newIgnore := make([]string, 0)
copy(newIgnore, ignore)
hash, err := HashDir(p, newIgnore)
if err != nil {
panic(err)
}
hashArray = append(hashArray, hash)
} else {
hash, err := HashFile(p)
if err != nil {
panic(err)
}
hashArray = append(hashArray, hash)
}
}

hash := crypto.SHA256.New()
hash.Write([]byte(path))
for _, v := range hashArray {
_, err := hash.Write([]byte(v))
if err != nil {
panic(err)
}
}
res := string(hash.Sum(nil))

// return it

return res, nil
}

func HashFile(path string) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()

hash := md5.New()
_, err = io.Copy(hash, file)
if err != nil {
return "", err
}
sum := string(hash.Sum(nil))

return sum, nil
}

func ReadIgnore(path string, ignore *[]string) error {
if ignore == nil {
panic("ignore slice is empty")
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()

scanner := bufio.NewScanner(file)
for scanner.Scan() {
*ignore = append(*ignore, scanner.Text())
}

if err := scanner.Err(); err != nil {
log.Fatal(err)
}

return nil
}
15 changes: 15 additions & 0 deletions dirhash/dirhash_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package dirhash

import (
"fmt"
"optimus/utils"
"testing"
)

func TestDirHash(t *testing.T) {
s, err := HashDir(utils.ProjectRoot(), make([]string, 0))
if err != nil {
t.Error(err)
}
fmt.Println(s)
}
17 changes: 17 additions & 0 deletions dirhash/read_ignore_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package dirhash

import (
"fmt"
"optimus/utils"
"testing"
)

func TestReadIgnore(t *testing.T) {
ignores := make([]string, 0)
err := ReadIgnore(utils.ProjectRoot()+"/.gitignore", &ignores)
if err != nil {
t.Error(err)
}

fmt.Println(ignores)
}
2 changes: 1 addition & 1 deletion flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

src = ./.;

vendorHash = "sha256-3tO/+Mnvl/wpS7Ro3XDIVrlYTGVM680mcC15/7ON6qM=";
vendorHash = "sha256-qWB4wz4JfxEh4LiixD5JK8/mmF/kmEKwTLh4mqEdDbA=";
# vendorHash = pkgs.lib.fakeHash;

meta = with pkgs.lib; {
Expand Down
11 changes: 9 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,29 @@ module optimus

go 1.20

require (
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.16.0
google.golang.org/appengine v1.6.7
)

require (
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.16.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
11 changes: 11 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
Expand All @@ -96,6 +99,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
Expand Down Expand Up @@ -247,6 +251,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -394,6 +400,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
Expand Down Expand Up @@ -457,6 +464,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
Expand Down
24 changes: 24 additions & 0 deletions lockfile/lockfile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package lockfile

import "optimus/utils"

type Lockfile struct {
ProjectHash string
Services []ServiceTestLock
}

type ServiceTestLock struct {
Name string
TestResult bool
Hash string
DependsOn []struct {
Name string
Hash string
}
}

func LoadLockfile() Lockfile {
_ = utils.ProjectRoot()

return Lockfile{}
}
13 changes: 3 additions & 10 deletions optimus.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,10 @@ e2e_tests:
# file: ./e2e_tests.nu

services:
frontend:
root: ./config
dev:
shell: nu -c
run: |
ls
optimus:
root: .
test: |
echo "testing"
# false
build: |
ls
go test ./...
lscmd:
root: ./config
Expand Down

0 comments on commit b6e869c

Please sign in to comment.