From 09839f5c4eb3465f1838f6e41a633539f79c5281 Mon Sep 17 00:00:00 2001 From: Tom van der Woerdt Date: Wed, 28 Aug 2024 14:10:44 +0200 Subject: [PATCH 1/5] lint: run 'go fmt' --- ansi/ansi_test.go | 2 +- ansi/tty.go | 1 + ansi/tty_windows.go | 2 +- api/api.go | 6 +- api/api_test.go | 32 +- client/coalesce.go | 4 +- codesearch/index/mmap_bsd.go | 51 +-- codesearch/index/mmap_linux.go | 50 +-- codesearch/index/mmap_windows.go | 2 +- codesearch/index/read.go | 548 +++++++++++++++---------------- codesearch/index/write.go | 18 +- codesearch/regexp/match.go | 6 +- codesearch/regexp/utf.go | 2 +- config/config_test.go | 4 +- index/grep.go | 6 +- index/index_test.go | 6 +- searcher/searcher.go | 3 +- vcs/git.go | 8 +- vcs/vcs.go | 1 - vcs/vcs_test.go | 2 +- 20 files changed, 377 insertions(+), 377 deletions(-) diff --git a/ansi/ansi_test.go b/ansi/ansi_test.go index 10456c13..eb1490da 100644 --- a/ansi/ansi_test.go +++ b/ansi/ansi_test.go @@ -11,7 +11,7 @@ var ( printTests = false ) -func makeReal(s string) string { //nolint +func makeReal(s string) string { //nolint return strings.Replace(s, "~", "\x1b", -1) } diff --git a/ansi/tty.go b/ansi/tty.go index 2597e681..b4b09a98 100644 --- a/ansi/tty.go +++ b/ansi/tty.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package ansi diff --git a/ansi/tty_windows.go b/ansi/tty_windows.go index 174a8c81..5ac64c58 100644 --- a/ansi/tty_windows.go +++ b/ansi/tty_windows.go @@ -6,7 +6,7 @@ import ( ) var ( - modkernel32 = syscall.MustLoadDLL("kernel32.dll") + modkernel32 = syscall.MustLoadDLL("kernel32.dll") procGetConsoleMode = modkernel32.MustFindProc("GetConsoleMode") ) diff --git a/api/api.go b/api/api.go index b7aef0eb..f32c33d5 100644 --- a/api/api.go +++ b/api/api.go @@ -90,7 +90,7 @@ func searchAll( *filesOpened += r.res.FilesOpened } - *duration = int(time.Now().Sub(startedAt).Seconds() * 1000) //nolint + *duration = int(time.Now().Sub(startedAt).Seconds() * 1000) //nolint return res, nil } @@ -281,7 +281,7 @@ func Setup(m *http.ServeMux, idx map[string]*searcher.Searcher, defaultMaxResult type Webhook struct { Repository struct { - Name string + Name string Full_name string } } @@ -291,7 +291,7 @@ func Setup(m *http.ServeMux, idx map[string]*searcher.Searcher, defaultMaxResult err := json.NewDecoder(r.Body).Decode(&h) if err != nil { - writeError(w, + writeError(w, errors.New(http.StatusText(http.StatusBadRequest)), http.StatusBadRequest) return diff --git a/api/api_test.go b/api/api_test.go index de072a8e..98da9023 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -6,29 +6,29 @@ import ( var parseAsIntAndUintTests = map[string]struct { numResults string - min int - max int - def int - expected int -} { - "parse error test case": {"not a parsable integer", 101, 1000, 5000, 5000}, - "less than min test case": {"100", 101, 1000, 5000, 101}, + min int + max int + def int + expected int +}{ + "parse error test case": {"not a parsable integer", 101, 1000, 5000, 5000}, + "less than min test case": {"100", 101, 1000, 5000, 101}, "greater than max test case": {"1001", 101, 1000, 5000, 1000}, - "within limits test case": {"100", 0, 100, 5000, 100}, + "within limits test case": {"100", 0, 100, 5000, 100}, } func TestParseAsIntAndUint(t *testing.T) { for name, td := range parseAsIntAndUintTests { t.Run(name, func(t *testing.T) { - if got, expected := + if got, expected := parseAsUintValue(td.numResults, uint(td.min), uint(td.max), uint(td.def)), uint(td.expected); got != expected { - t.Fatalf("parseAsUintValue - %s: returned %d; expected %d", name, got, expected) - } + t.Fatalf("parseAsUintValue - %s: returned %d; expected %d", name, got, expected) + } - if got, expected := + if got, expected := parseAsIntValue(td.numResults, td.min, td.max, td.def), td.expected; got != expected { - t.Fatalf("parseAsIntValue - %s: returned %d; expected %d", name, got, expected) - } - }) + t.Fatalf("parseAsIntValue - %s: returned %d; expected %d", name, got, expected) + } + }) } -} \ No newline at end of file +} diff --git a/client/coalesce.go b/client/coalesce.go index 0c55794c..73ec7842 100644 --- a/client/coalesce.go +++ b/client/coalesce.go @@ -30,13 +30,13 @@ func matchToBlock(m *index.Match) *Block { v[b] = true - for _, line := range m.Before { //nolint + for _, line := range m.Before { //nolint l = append(l, line) } l = append(l, m.Line) - for _, line := range m.After { //nolint + for _, line := range m.After { //nolint l = append(l, line) } diff --git a/codesearch/index/mmap_bsd.go b/codesearch/index/mmap_bsd.go index 9442ab58..b0fecee7 100644 --- a/codesearch/index/mmap_bsd.go +++ b/codesearch/index/mmap_bsd.go @@ -2,44 +2,45 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build darwin || freebsd || openbsd || netbsd // +build darwin freebsd openbsd netbsd package index import ( - "log" - "os" - "syscall" + "log" + "os" + "syscall" ) func mmapFile(f *os.File) mmapData { - st, err := f.Stat() - if err != nil { - log.Fatal(err) - } - size := st.Size() - if int64(int(size+4095)) != size+4095 { - log.Fatalf("%s: too large for mmap", f.Name()) - } - n := int(size) - if n == 0 { - return mmapData{f, nil, nil} - } - data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_PRIVATE) - if err != nil { - log.Fatalf("mmap %s: %v", f.Name(), err) - } - return mmapData{f, data[:n], data} + st, err := f.Stat() + if err != nil { + log.Fatal(err) + } + size := st.Size() + if int64(int(size+4095)) != size+4095 { + log.Fatalf("%s: too large for mmap", f.Name()) + } + n := int(size) + if n == 0 { + return mmapData{f, nil, nil} + } + data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_PRIVATE) + if err != nil { + log.Fatalf("mmap %s: %v", f.Name(), err) + } + return mmapData{f, data[:n], data} } func unmmapFile(m *mmapData) error { - if err := syscall.Munmap(m.o); err != nil { - return err - } + if err := syscall.Munmap(m.o); err != nil { + return err + } - return m.f.Close() + return m.f.Close() } func unmmap(d []byte) error { - return syscall.Munmap(d) + return syscall.Munmap(d) } diff --git a/codesearch/index/mmap_linux.go b/codesearch/index/mmap_linux.go index 8ea561a3..dd414038 100644 --- a/codesearch/index/mmap_linux.go +++ b/codesearch/index/mmap_linux.go @@ -5,39 +5,39 @@ package index import ( - "log" - "os" - "syscall" + "log" + "os" + "syscall" ) func mmapFile(f *os.File) mmapData { - st, err := f.Stat() - if err != nil { - log.Fatal(err) - } - size := st.Size() - if int64(int(size+4095)) != size+4095 { - log.Fatalf("%s: too large for mmap", f.Name()) - } - n := int(size) - if n == 0 { - return mmapData{f, nil, nil} - } - data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - log.Fatalf("mmap %s: %v", f.Name(), err) - } - return mmapData{f, data[:n], data} + st, err := f.Stat() + if err != nil { + log.Fatal(err) + } + size := st.Size() + if int64(int(size+4095)) != size+4095 { + log.Fatalf("%s: too large for mmap", f.Name()) + } + n := int(size) + if n == 0 { + return mmapData{f, nil, nil} + } + data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + log.Fatalf("mmap %s: %v", f.Name(), err) + } + return mmapData{f, data[:n], data} } func unmmapFile(m *mmapData) error { - if err := syscall.Munmap(m.o); err != nil { - return err - } + if err := syscall.Munmap(m.o); err != nil { + return err + } - return m.f.Close() + return m.f.Close() } func unmmap(d []byte) error { - return syscall.Munmap(d) + return syscall.Munmap(d) } diff --git a/codesearch/index/mmap_windows.go b/codesearch/index/mmap_windows.go index 8f091562..a5ca5d2d 100644 --- a/codesearch/index/mmap_windows.go +++ b/codesearch/index/mmap_windows.go @@ -49,4 +49,4 @@ func unmmapFile(m *mmapData) error { func unmmap(d []byte) error { return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&d))) -} \ No newline at end of file +} diff --git a/codesearch/index/read.go b/codesearch/index/read.go index 5234266e..a6ceb5df 100644 --- a/codesearch/index/read.go +++ b/codesearch/index/read.go @@ -65,387 +65,387 @@ package index // "\ncsearch trailr\n" import ( - "bytes" - "encoding/binary" - "log" - "os" - "runtime" - "sort" + "bytes" + "encoding/binary" + "log" + "os" + "runtime" + "sort" ) const ( - magic = "csearch index 1\n" - trailerMagic = "\ncsearch trailr\n" + magic = "csearch index 1\n" + trailerMagic = "\ncsearch trailr\n" ) // An Index implements read-only access to a trigram index. type Index struct { - Verbose bool - data mmapData - pathData uint32 - nameData uint32 - postData uint32 - nameIndex uint32 - postIndex uint32 - numName int - numPost int + Verbose bool + data mmapData + pathData uint32 + nameData uint32 + postData uint32 + nameIndex uint32 + postIndex uint32 + numName int + numPost int } const postEntrySize = 3 + 4 + 4 func Open(file string) *Index { - mm := mmap(file) - if len(mm.d) < 4*4+len(trailerMagic) || string(mm.d[len(mm.d)-len(trailerMagic):]) != trailerMagic { - corrupt(mm.f) - } - n := uint32(len(mm.d) - len(trailerMagic) - 5*4) - ix := &Index{data: mm} - ix.pathData = ix.uint32(n) - ix.nameData = ix.uint32(n + 4) - ix.postData = ix.uint32(n + 8) - ix.nameIndex = ix.uint32(n + 12) - ix.postIndex = ix.uint32(n + 16) - ix.numName = int((ix.postIndex-ix.nameIndex)/4) - 1 - ix.numPost = int((n - ix.postIndex) / postEntrySize) - return ix + mm := mmap(file) + if len(mm.d) < 4*4+len(trailerMagic) || string(mm.d[len(mm.d)-len(trailerMagic):]) != trailerMagic { + corrupt(mm.f) + } + n := uint32(len(mm.d) - len(trailerMagic) - 5*4) + ix := &Index{data: mm} + ix.pathData = ix.uint32(n) + ix.nameData = ix.uint32(n + 4) + ix.postData = ix.uint32(n + 8) + ix.nameIndex = ix.uint32(n + 12) + ix.postIndex = ix.uint32(n + 16) + ix.numName = int((ix.postIndex-ix.nameIndex)/4) - 1 + ix.numPost = int((n - ix.postIndex) / postEntrySize) + return ix } // slice returns the slice of index data starting at the given byte offset. // If n >= 0, the slice must have length at least n and is truncated to length n. func (ix *Index) slice(off uint32, n int) []byte { - o := int(off) - if uint32(o) != off || n >= 0 && o+n > len(ix.data.d) { - corrupt(ix.data.f) - } - if n < 0 { - return ix.data.d[o:] - } - return ix.data.d[o : o+n] + o := int(off) + if uint32(o) != off || n >= 0 && o+n > len(ix.data.d) { + corrupt(ix.data.f) + } + if n < 0 { + return ix.data.d[o:] + } + return ix.data.d[o : o+n] } // uint32 returns the uint32 value at the given offset in the index data. func (ix *Index) uint32(off uint32) uint32 { - return binary.BigEndian.Uint32(ix.slice(off, 4)) + return binary.BigEndian.Uint32(ix.slice(off, 4)) } func (ix *Index) Close() error { - return ix.data.close() + return ix.data.close() } // uvarint returns the varint value at the given offset in the index data. -func (ix *Index) uvarint(off uint32) uint32 { //nolint - v, n := binary.Uvarint(ix.slice(off, -1)) - if n <= 0 { - corrupt(ix.data.f) - } - return uint32(v) +func (ix *Index) uvarint(off uint32) uint32 { //nolint + v, n := binary.Uvarint(ix.slice(off, -1)) + if n <= 0 { + corrupt(ix.data.f) + } + return uint32(v) } // Paths returns the list of indexed paths. func (ix *Index) Paths() []string { - off := ix.pathData - var x []string - for { - s := ix.str(off) - if len(s) == 0 { - break - } - x = append(x, string(s)) - off += uint32(len(s) + 1) - } - return x + off := ix.pathData + var x []string + for { + s := ix.str(off) + if len(s) == 0 { + break + } + x = append(x, string(s)) + off += uint32(len(s) + 1) + } + return x } // NameBytes returns the name corresponding to the given fileid. func (ix *Index) NameBytes(fileid uint32) []byte { - off := ix.uint32(ix.nameIndex + 4*fileid) - return ix.str(ix.nameData + off) + off := ix.uint32(ix.nameIndex + 4*fileid) + return ix.str(ix.nameData + off) } func (ix *Index) str(off uint32) []byte { - str := ix.slice(off, -1) - i := bytes.IndexByte(str, '\x00') - if i < 0 { - corrupt(ix.data.f) - } - return str[:i] + str := ix.slice(off, -1) + i := bytes.IndexByte(str, '\x00') + if i < 0 { + corrupt(ix.data.f) + } + return str[:i] } // Name returns the name corresponding to the given fileid. func (ix *Index) Name(fileid uint32) string { - return string(ix.NameBytes(fileid)) + return string(ix.NameBytes(fileid)) } // listAt returns the index list entry at the given offset. func (ix *Index) listAt(off uint32) (trigram, count, offset uint32) { - d := ix.slice(ix.postIndex+off, postEntrySize) - trigram = uint32(d[0])<<16 | uint32(d[1])<<8 | uint32(d[2]) - count = binary.BigEndian.Uint32(d[3:]) - offset = binary.BigEndian.Uint32(d[3+4:]) - return + d := ix.slice(ix.postIndex+off, postEntrySize) + trigram = uint32(d[0])<<16 | uint32(d[1])<<8 | uint32(d[2]) + count = binary.BigEndian.Uint32(d[3:]) + offset = binary.BigEndian.Uint32(d[3+4:]) + return } -func (ix *Index) dumpPosting() { //nolint - d := ix.slice(ix.postIndex, postEntrySize*ix.numPost) - for i := 0; i < ix.numPost; i++ { - j := i * postEntrySize - t := uint32(d[j])<<16 | uint32(d[j+1])<<8 | uint32(d[j+2]) - count := int(binary.BigEndian.Uint32(d[j+3:])) - offset := binary.BigEndian.Uint32(d[j+3+4:]) - log.Printf("%#x: %d at %d", t, count, offset) - } +func (ix *Index) dumpPosting() { //nolint + d := ix.slice(ix.postIndex, postEntrySize*ix.numPost) + for i := 0; i < ix.numPost; i++ { + j := i * postEntrySize + t := uint32(d[j])<<16 | uint32(d[j+1])<<8 | uint32(d[j+2]) + count := int(binary.BigEndian.Uint32(d[j+3:])) + offset := binary.BigEndian.Uint32(d[j+3+4:]) + log.Printf("%#x: %d at %d", t, count, offset) + } } func (ix *Index) findList(trigram uint32) (count int, offset uint32) { - // binary search - d := ix.slice(ix.postIndex, postEntrySize*ix.numPost) - i := sort.Search(ix.numPost, func(i int) bool { - i *= postEntrySize - t := uint32(d[i])<<16 | uint32(d[i+1])<<8 | uint32(d[i+2]) - return t >= trigram - }) - if i >= ix.numPost { - return 0, 0 - } - i *= postEntrySize - t := uint32(d[i])<<16 | uint32(d[i+1])<<8 | uint32(d[i+2]) - if t != trigram { - return 0, 0 - } - count = int(binary.BigEndian.Uint32(d[i+3:])) - offset = binary.BigEndian.Uint32(d[i+3+4:]) - return + // binary search + d := ix.slice(ix.postIndex, postEntrySize*ix.numPost) + i := sort.Search(ix.numPost, func(i int) bool { + i *= postEntrySize + t := uint32(d[i])<<16 | uint32(d[i+1])<<8 | uint32(d[i+2]) + return t >= trigram + }) + if i >= ix.numPost { + return 0, 0 + } + i *= postEntrySize + t := uint32(d[i])<<16 | uint32(d[i+1])<<8 | uint32(d[i+2]) + if t != trigram { + return 0, 0 + } + count = int(binary.BigEndian.Uint32(d[i+3:])) + offset = binary.BigEndian.Uint32(d[i+3+4:]) + return } type postReader struct { - ix *Index - count int - offset uint32 - fileid uint32 - d []byte - restrict []uint32 + ix *Index + count int + offset uint32 + fileid uint32 + d []byte + restrict []uint32 } func (r *postReader) init(ix *Index, trigram uint32, restrict []uint32) { - count, offset := ix.findList(trigram) - if count == 0 { - return - } - r.ix = ix - r.count = count - r.offset = offset - r.fileid = ^uint32(0) - r.d = ix.slice(ix.postData+offset+3, -1) - r.restrict = restrict + count, offset := ix.findList(trigram) + if count == 0 { + return + } + r.ix = ix + r.count = count + r.offset = offset + r.fileid = ^uint32(0) + r.d = ix.slice(ix.postData+offset+3, -1) + r.restrict = restrict } func (r *postReader) max() int { - return int(r.count) + return int(r.count) } func (r *postReader) next() bool { - for r.count > 0 { - r.count-- - delta64, n := binary.Uvarint(r.d) - delta := uint32(delta64) - if n <= 0 || delta == 0 { - corrupt(r.ix.data.f) - } - r.d = r.d[n:] - r.fileid += delta - if r.restrict != nil { - i := 0 - for i < len(r.restrict) && r.restrict[i] < r.fileid { - i++ - } - r.restrict = r.restrict[i:] - if len(r.restrict) == 0 || r.restrict[0] != r.fileid { - continue - } - } - return true - } - // list should end with terminating 0 delta - if r.d != nil && (len(r.d) == 0 || r.d[0] != 0) { - corrupt(r.ix.data.f) - } - r.fileid = ^uint32(0) - return false + for r.count > 0 { + r.count-- + delta64, n := binary.Uvarint(r.d) + delta := uint32(delta64) + if n <= 0 || delta == 0 { + corrupt(r.ix.data.f) + } + r.d = r.d[n:] + r.fileid += delta + if r.restrict != nil { + i := 0 + for i < len(r.restrict) && r.restrict[i] < r.fileid { + i++ + } + r.restrict = r.restrict[i:] + if len(r.restrict) == 0 || r.restrict[0] != r.fileid { + continue + } + } + return true + } + // list should end with terminating 0 delta + if r.d != nil && (len(r.d) == 0 || r.d[0] != 0) { + corrupt(r.ix.data.f) + } + r.fileid = ^uint32(0) + return false } func (ix *Index) PostingList(trigram uint32) []uint32 { - return ix.postingList(trigram, nil) + return ix.postingList(trigram, nil) } func (ix *Index) postingList(trigram uint32, restrict []uint32) []uint32 { - var r postReader - r.init(ix, trigram, restrict) - x := make([]uint32, 0, r.max()) - for r.next() { - x = append(x, r.fileid) - } - return x + var r postReader + r.init(ix, trigram, restrict) + x := make([]uint32, 0, r.max()) + for r.next() { + x = append(x, r.fileid) + } + return x } func (ix *Index) PostingAnd(list []uint32, trigram uint32) []uint32 { - return ix.postingAnd(list, trigram, nil) + return ix.postingAnd(list, trigram, nil) } func (ix *Index) postingAnd(list []uint32, trigram uint32, restrict []uint32) []uint32 { - var r postReader - r.init(ix, trigram, restrict) - x := list[:0] - i := 0 - for r.next() { - fileid := r.fileid - for i < len(list) && list[i] < fileid { - i++ - } - if i < len(list) && list[i] == fileid { - x = append(x, fileid) - i++ - } - } - return x + var r postReader + r.init(ix, trigram, restrict) + x := list[:0] + i := 0 + for r.next() { + fileid := r.fileid + for i < len(list) && list[i] < fileid { + i++ + } + if i < len(list) && list[i] == fileid { + x = append(x, fileid) + i++ + } + } + return x } func (ix *Index) PostingOr(list []uint32, trigram uint32) []uint32 { - return ix.postingOr(list, trigram, nil) + return ix.postingOr(list, trigram, nil) } func (ix *Index) postingOr(list []uint32, trigram uint32, restrict []uint32) []uint32 { - var r postReader - r.init(ix, trigram, restrict) - x := make([]uint32, 0, len(list)+r.max()) - i := 0 - for r.next() { - fileid := r.fileid - for i < len(list) && list[i] < fileid { - x = append(x, list[i]) - i++ - } - x = append(x, fileid) - if i < len(list) && list[i] == fileid { - i++ - } - } - x = append(x, list[i:]...) - return x + var r postReader + r.init(ix, trigram, restrict) + x := make([]uint32, 0, len(list)+r.max()) + i := 0 + for r.next() { + fileid := r.fileid + for i < len(list) && list[i] < fileid { + x = append(x, list[i]) + i++ + } + x = append(x, fileid) + if i < len(list) && list[i] == fileid { + i++ + } + } + x = append(x, list[i:]...) + return x } func (ix *Index) PostingQuery(q *Query) []uint32 { - return ix.postingQuery(q, nil) + return ix.postingQuery(q, nil) } func (ix *Index) postingQuery(q *Query, restrict []uint32) (ret []uint32) { - var list []uint32 - switch q.Op { - case QNone: - // nothing - case QAll: - if restrict != nil { - return restrict - } - list = make([]uint32, ix.numName) - for i := range list { - list[i] = uint32(i) - } - return list - case QAnd: - for _, t := range q.Trigram { - tri := uint32(t[0])<<16 | uint32(t[1])<<8 | uint32(t[2]) - if list == nil { - list = ix.postingList(tri, restrict) - } else { - list = ix.postingAnd(list, tri, restrict) - } - if len(list) == 0 { - return nil - } - } - for _, sub := range q.Sub { - if list == nil { - list = restrict - } - list = ix.postingQuery(sub, list) - if len(list) == 0 { - return nil - } - } - case QOr: - for _, t := range q.Trigram { - tri := uint32(t[0])<<16 | uint32(t[1])<<8 | uint32(t[2]) - if list == nil { - list = ix.postingList(tri, restrict) - } else { - list = ix.postingOr(list, tri, restrict) - } - } - for _, sub := range q.Sub { - list1 := ix.postingQuery(sub, restrict) - list = mergeOr(list, list1) - } - } - return list + var list []uint32 + switch q.Op { + case QNone: + // nothing + case QAll: + if restrict != nil { + return restrict + } + list = make([]uint32, ix.numName) + for i := range list { + list[i] = uint32(i) + } + return list + case QAnd: + for _, t := range q.Trigram { + tri := uint32(t[0])<<16 | uint32(t[1])<<8 | uint32(t[2]) + if list == nil { + list = ix.postingList(tri, restrict) + } else { + list = ix.postingAnd(list, tri, restrict) + } + if len(list) == 0 { + return nil + } + } + for _, sub := range q.Sub { + if list == nil { + list = restrict + } + list = ix.postingQuery(sub, list) + if len(list) == 0 { + return nil + } + } + case QOr: + for _, t := range q.Trigram { + tri := uint32(t[0])<<16 | uint32(t[1])<<8 | uint32(t[2]) + if list == nil { + list = ix.postingList(tri, restrict) + } else { + list = ix.postingOr(list, tri, restrict) + } + } + for _, sub := range q.Sub { + list1 := ix.postingQuery(sub, restrict) + list = mergeOr(list, list1) + } + } + return list } func mergeOr(l1, l2 []uint32) []uint32 { - var l []uint32 - i := 0 - j := 0 - for i < len(l1) || j < len(l2) { - switch { - case j == len(l2) || (i < len(l1) && l1[i] < l2[j]): - l = append(l, l1[i]) - i++ - case i == len(l1) || (j < len(l2) && l1[i] > l2[j]): - l = append(l, l2[j]) - j++ - case l1[i] == l2[j]: - l = append(l, l1[i]) - i++ - j++ - } - } - return l + var l []uint32 + i := 0 + j := 0 + for i < len(l1) || j < len(l2) { + switch { + case j == len(l2) || (i < len(l1) && l1[i] < l2[j]): + l = append(l, l1[i]) + i++ + case i == len(l1) || (j < len(l2) && l1[i] > l2[j]): + l = append(l, l2[j]) + j++ + case l1[i] == l2[j]: + l = append(l, l1[i]) + i++ + j++ + } + } + return l } func corrupt(file *os.File) { - log.Fatalf("corrupt index: %s", file.Name()) + log.Fatalf("corrupt index: %s", file.Name()) } // An mmapData is mmap'ed read-only data from a file. type mmapData struct { - f *os.File - d []byte - o []byte + f *os.File + d []byte + o []byte } func (m *mmapData) close() error { - return unmmapFile(m) + return unmmapFile(m) } // mmap maps the given file into memory. func mmap(file string) mmapData { - f, err := os.Open(file) - if err != nil { - log.Fatal(err) - } - return mmapFile(f) + f, err := os.Open(file) + if err != nil { + log.Fatal(err) + } + return mmapFile(f) } // File returns the name of the index file to use. // It is either $CSEARCHINDEX or $HOME/.csearchindex. func File() string { - f := os.Getenv("CSEARCHINDEX") - if f != "" { - return f - } - var home string - if runtime.GOOS == "windows" { - home = os.Getenv("HOMEPATH") - } else { - home = os.Getenv("HOME") - } - return home + "/.csearchindex" + f := os.Getenv("CSEARCHINDEX") + if f != "" { + return f + } + var home string + if runtime.GOOS == "windows" { + home = os.Getenv("HOMEPATH") + } else { + home = os.Getenv("HOME") + } + return home + "/.csearchindex" } diff --git a/codesearch/index/write.go b/codesearch/index/write.go index f056e35e..0f3fee01 100644 --- a/codesearch/index/write.go +++ b/codesearch/index/write.go @@ -123,7 +123,7 @@ func (ix *IndexWriter) AddFile(name string) { func (ix *IndexWriter) Add(name string, f io.Reader) string { ix.trigram.Reset() var ( - c = byte(0) //nolint + c = byte(0) //nolint i = 0 buf = ix.inbuf[:0] tv = uint32(0) @@ -131,7 +131,7 @@ func (ix *IndexWriter) Add(name string, f io.Reader) string { linelen = 0 numLines = 0 longLines = 0 - skipReason = "" //nolint + skipReason = "" //nolint ) for { @@ -246,7 +246,7 @@ func (ix *IndexWriter) Flush() { os.Remove(ix.nameData.name) for _, d := range ix.postData { - unmmap(d) //nolint + unmmap(d) //nolint } for _, f := range ix.postFile { f.Close() @@ -310,7 +310,7 @@ func (ix *IndexWriter) flushPost() { } ix.post = ix.post[:0] - w.Seek(0, 0) //nolint + w.Seek(0, 0) //nolint ix.postFile = append(ix.postFile, w) } @@ -368,7 +368,7 @@ type postChunk struct { m []postEntry // remaining entries after e } -const postBuf = 4096 //nolint +const postBuf = 4096 //nolint // A postHeap is a heap (priority queue) of postChunks. type postHeap struct { @@ -388,7 +388,7 @@ func (h *postHeap) addMem(x []postEntry) { // step reads the next entry from ch and saves it in ch.e. // It returns false if ch is over. -func (h *postHeap) step(ch *postChunk) bool { //nolint +func (h *postHeap) step(ch *postChunk) bool { //nolint old := ch.e m := ch.m if len(m) == 0 { @@ -414,7 +414,7 @@ func (h *postHeap) add(ch *postChunk) { } // empty reports whether the postHeap is empty. -func (h *postHeap) empty() bool { //nolint +func (h *postHeap) empty() bool { //nolint return len(h.ch) == 0 } @@ -492,7 +492,7 @@ type bufWriter struct { name string file *os.File buf []byte - tmp [8]byte //nolint + tmp [8]byte //nolint } // bufCreate creates a new file with the given name and returns a @@ -578,7 +578,7 @@ func (b *bufWriter) flush() { func (b *bufWriter) finish() *os.File { b.flush() f := b.file - f.Seek(0, 0) //nolint + f.Seek(0, 0) //nolint return f } diff --git a/codesearch/regexp/match.go b/codesearch/regexp/match.go index 480f9109..d00e2f88 100644 --- a/codesearch/regexp/match.go +++ b/codesearch/regexp/match.go @@ -338,9 +338,9 @@ func (m *matcher) matchString(b string, beginText, endText bool) (end int) { // isWordByte reports whether the byte c is a word character: ASCII only. // This is used to implement \b and \B. This is not right for Unicode, but: -// - it's hard to get right in a byte-at-a-time matching world -// (the DFA has only one-byte lookahead) -// - this crude approximation is the same one PCRE uses +// - it's hard to get right in a byte-at-a-time matching world +// (the DFA has only one-byte lookahead) +// - this crude approximation is the same one PCRE uses func isWordByte(c int) bool { return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || diff --git a/codesearch/regexp/utf.go b/codesearch/regexp/utf.go index 858c83c1..1085df59 100644 --- a/codesearch/regexp/utf.go +++ b/codesearch/regexp/utf.go @@ -216,7 +216,7 @@ func (b *runeBuilder) addRange(lo, hi rune, fold bool) { } // TODO: Pick off 80-10FFFF for special handling? - if lo == 0x80 && hi == 0x10FFFF { //nolint + if lo == 0x80 && hi == 0x10FFFF { //nolint } // Split range into same-length sized ranges. diff --git a/config/config_test.go b/config/config_test.go index 5b3328af..22d38855 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -40,7 +40,7 @@ func TestExampleConfigsAreValid(t *testing.T) { repo := cfg.Repos["SomeGitRepo"] vcsConfigBytes := repo.VcsConfig() var vcsConfigVals map[string]interface{} - json.Unmarshal(vcsConfigBytes, &vcsConfigVals) //nolint + json.Unmarshal(vcsConfigBytes, &vcsConfigVals) //nolint if detectRef, ok := vcsConfigVals["detect-ref"]; !ok || !detectRef.(bool) { t.Error("global detectRef vcs config setting not set for repo") } @@ -51,7 +51,7 @@ func TestExampleConfigsAreValid(t *testing.T) { repo = cfg.Repos["GitRepoWithDetectRefDisabled"] vcsConfigBytes = repo.VcsConfig() - json.Unmarshal(vcsConfigBytes, &vcsConfigVals) //nolint + json.Unmarshal(vcsConfigBytes, &vcsConfigVals) //nolint if detectRef, ok := vcsConfigVals["detect-ref"]; !ok || detectRef.(bool) { t.Error("global detectRef vcs config setting not overriden by repo-level setting") } diff --git a/index/grep.go b/index/grep.go index d8af1640..641914aa 100644 --- a/index/grep.go +++ b/index/grep.go @@ -28,7 +28,7 @@ func countLines(b []byte) int { return n } -func (g *grepper) grepFile(filename string, re *regexp.Regexp, //nolint +func (g *grepper) grepFile(filename string, re *regexp.Regexp, //nolint fn func(line []byte, lineno int) (bool, error)) error { r, err := os.Open(filename) if err != nil { @@ -189,7 +189,7 @@ func (g *grepper) grep2( // This nonsense is adapted from https://code.google.com/p/codesearch/source/browse/regexp/match.go#399 // and I assume it is a mess to make it faster, but I would like to try a much simpler cleaner version. -func (g *grepper) grep(r io.Reader, re *regexp.Regexp, fn func(line []byte, lineno int) (bool, error)) error { //nolint +func (g *grepper) grep(r io.Reader, re *regexp.Regexp, fn func(line []byte, lineno int) (bool, error)) error { //nolint if g.buf == nil { g.buf = make([]byte, 1<<20) } @@ -248,5 +248,5 @@ func (g *grepper) grep(r io.Reader, re *regexp.Regexp, fn func(line []byte, line } } - return nil //nolint + return nil //nolint } diff --git a/index/index_test.go b/index/index_test.go index a8de1c34..7b61ce0a 100644 --- a/index/index_test.go +++ b/index/index_test.go @@ -37,7 +37,7 @@ func TestSearch(t *testing.T) { if err != nil { t.Fatal(err) } - defer ref.Remove() //nolint + defer ref.Remove() //nolint // Make sure the metadata in the ref is good. if ref.Rev != rev { @@ -67,7 +67,7 @@ func TestSearchWithLimits(t *testing.T) { if err != nil { t.Fatal(err) } - defer ref.Remove() //nolint + defer ref.Remove() //nolint // Make sure the ref can be opened. idx, err := ref.Open() @@ -116,7 +116,7 @@ func TestRead(t *testing.T) { if err != nil { t.Fatal(err) } - defer ref.Remove() //nolint + defer ref.Remove() //nolint r, err := Read(ref.Dir()) if err != nil { diff --git a/searcher/searcher.go b/searcher/searcher.go index 791ce810..13a0d1b7 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -264,7 +264,7 @@ func reportOnMemory() { // Utility function for producing a hex encoded sha1 hash for a string. func hashFor(name string) string { h := sha1.New() - h.Write([]byte(name)) //nolint + h.Write([]byte(name)) //nolint return hex.EncodeToString(h.Sum(nil)) } @@ -407,7 +407,6 @@ func newSearcher( return nil, err } - rev, err := wd.PullOrClone(vcsDir, repo.Url) if err != nil { return nil, err diff --git a/vcs/git.go b/vcs/git.go index 54f1dfc5..b488d5cc 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -155,9 +155,9 @@ func (g *GitDriver) SpecialFiles() []string { func (g *GitDriver) AutoGeneratedFiles(dir string) []string { var files []string - filesCmd := exec.Command("git", "ls-files", "-z"); + filesCmd := exec.Command("git", "ls-files", "-z") filesCmd.Dir = dir - pipe, err := filesCmd.StdoutPipe(); + pipe, err := filesCmd.StdoutPipe() if err != nil { log.Printf("Error occured when running git ls-files in %s: %s.", dir, err) @@ -169,7 +169,7 @@ func (g *GitDriver) AutoGeneratedFiles(dir string) []string { return files } - attributesCmd := exec.Command("git", "check-attr", "--stdin", "-z", autoGeneratedAttribute); + attributesCmd := exec.Command("git", "check-attr", "--stdin", "-z", autoGeneratedAttribute) attributesCmd.Dir = dir attributesCmd.Stdin = pipe @@ -183,7 +183,7 @@ func (g *GitDriver) AutoGeneratedFiles(dir string) []string { // Split by NUL and we expect the format: NUL NUL NUL tokens := bytes.Split(out, []byte{0}) - for i := 2; i < len(tokens); i+=3 { + for i := 2; i < len(tokens); i += 3 { if string(tokens[i]) == "true" && string(tokens[i-1]) == autoGeneratedAttribute { files = append(files, string(tokens[i-2])) } diff --git a/vcs/vcs.go b/vcs/vcs.go index 26f0d4d8..1e663274 100644 --- a/vcs/vcs.go +++ b/vcs/vcs.go @@ -29,7 +29,6 @@ type Driver interface { // Return a list of filenames that are marked as auto-generated. AutoGeneratedFiles(dir string) []string - } // An API to interact with a vcs working directory. This is diff --git a/vcs/vcs_test.go b/vcs/vcs_test.go index 5ef41e6b..a134ce68 100644 --- a/vcs/vcs_test.go +++ b/vcs/vcs_test.go @@ -8,7 +8,7 @@ import ( // Just make sure all drivers are tolerant of nil func TestNilConfigs(t *testing.T) { - for name, _ := range drivers { //nolint + for name, _ := range drivers { //nolint d, err := New(name, nil) if err != nil { t.Fatal(err) From a9acef6525cc78659d39af620ac722334ff49def Mon Sep 17 00:00:00 2001 From: Tom van der Woerdt Date: Wed, 28 Aug 2024 14:10:44 +0200 Subject: [PATCH 2/5] Add an abstraction layer 'FileSystem' to implement a VFS This just shuffles some APIs around, no real changes other than adding some indirection. --- index/index.go | 46 +++++++++++++------------------------------- index/index_test.go | 9 ++++++++- searcher/searcher.go | 21 +++++++++++++++----- vcs/bzr.go | 4 ++++ vcs/dir_fs.go | 44 ++++++++++++++++++++++++++++++++++++++++++ vcs/git.go | 4 ++++ vcs/hg.go | 4 ++++ vcs/local.go | 4 ++++ vcs/svn.go | 4 ++++ vcs/vcs.go | 12 ++++++++++++ 10 files changed, 113 insertions(+), 39 deletions(-) create mode 100644 vcs/dir_fs.go diff --git a/index/index.go b/index/index.go index f84d933b..e12ab3ec 100644 --- a/index/index.go +++ b/index/index.go @@ -13,6 +13,7 @@ import ( "github.com/hound-search/hound/codesearch/index" "github.com/hound-search/hound/codesearch/regexp" + "github.com/hound-search/hound/vcs" ) const ( @@ -256,9 +257,9 @@ func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error) }, nil } -func isTextFile(filename string) (bool, error) { +func isTextFile(filename string, fs vcs.FileSystem) (bool, error) { buf := make([]byte, filePeekSize) - r, err := os.Open(filename) + r, err := fs.Open(filename) if err != nil { return false, err } @@ -307,13 +308,8 @@ func validUTF8IgnoringPartialTrailingRune(p []byte) bool { return true } -func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error) { - rel, err := filepath.Rel(src, path) - if err != nil { - return "", err - } - - r, err := os.Open(path) +func addFileToIndex(ix *index.IndexWriter, dst string, fs vcs.FileSystem, rel string) (string, error) { + r, err := fs.Open(rel) if err != nil { return "", err } @@ -332,12 +328,7 @@ func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error return ix.Add(rel, io.TeeReader(r, g)), nil } -func addDirToIndex(dst, src, path string) error { - rel, err := filepath.Rel(src, path) - if err != nil { - return err - } - +func addDirToIndex(dst string, fs vcs.FileSystem, rel string) error { if rel == "." { return nil } @@ -366,7 +357,7 @@ func containsString(haystack []string, needle string) bool { return false } -func indexAllFiles(opt *IndexOptions, dst, src string) error { +func indexAllFiles(opt *IndexOptions, dst string, fs vcs.FileSystem) error { ix := index.Create(filepath.Join(dst, "tri")) defer ix.Close() @@ -379,19 +370,8 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error { } defer fileHandle.Close() - // Resolve the symbolic link - if fi, err := os.Stat(src); err == nil && fi.Mode()|os.ModeSymlink != 0 { - if s, err := os.Readlink(src); err == nil { - src = s - } - } - - if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { //nolint + if err := fs.Walk(func(rel string, info os.FileInfo, err error) error { //nolint name := info.Name() - rel, err := filepath.Rel(src, path) //nolint - if err != nil { - return err - } // Is this file considered "special", this means it's not even a part // of the source repository (like .git or .svn). @@ -415,7 +395,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error { } if info.IsDir() { - return addDirToIndex(dst, src, path) + return addDirToIndex(dst, fs, rel) } if info.Mode()&os.ModeType != 0 { @@ -426,7 +406,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error { return nil } - txt, err := isTextFile(path) + txt, err := isTextFile(rel, fs) if err != nil { return err } @@ -439,7 +419,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error { return nil } - reasonForExclusion, err := addFileToIndex(ix, dst, src, path) + reasonForExclusion, err := addFileToIndex(ix, dst, fs, rel) if err != nil { return err } @@ -485,7 +465,7 @@ func Read(dir string) (*IndexRef, error) { return m, nil } -func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) { +func Build(opt *IndexOptions, dst string, fs vcs.FileSystem, url, rev string) (*IndexRef, error) { if _, err := os.Stat(dst); err != nil { if err := os.MkdirAll(dst, os.ModePerm); err != nil { return nil, err @@ -496,7 +476,7 @@ func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) { return nil, err } - if err := indexAllFiles(opt, dst, src); err != nil { + if err := indexAllFiles(opt, dst, fs); err != nil { return nil, err } diff --git a/index/index_test.go b/index/index_test.go index 7b61ce0a..9fc86c10 100644 --- a/index/index_test.go +++ b/index/index_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "runtime" "testing" + + "github.com/hound-search/hound/vcs" ) const ( @@ -26,9 +28,14 @@ func buildIndex(url, rev string) (*IndexRef, error) { return nil, err } + dirFs, err := vcs.NewDirFilesystem(thisDir()) + if err != nil { + return nil, err + } + var opt IndexOptions - return Build(&opt, dir, thisDir(), url, rev) + return Build(&opt, dir, dirFs, url, rev) } func TestSearch(t *testing.T) { diff --git a/searcher/searcher.go b/searcher/searcher.go index 13a0d1b7..5857a911 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -232,13 +232,13 @@ func findExistingRefs(dbpath string) (*foundRefs, error) { // one will be built. func buildAndOpenIndex( opt *index.IndexOptions, - dbpath, - vcsDir, + dbpath string, + fs vcs.FileSystem, idxDir, url, rev string) (*index.Index, error) { if _, err := os.Stat(idxDir); err != nil { - r, err := index.Build(opt, idxDir, vcsDir, url, rev) + r, err := index.Build(opt, idxDir, fs, url, rev) if err != nil { return nil, err } @@ -366,11 +366,17 @@ func updateAndReindex( return rev, false } + fs, err := wd.FileSystem(vcsDir) + if err != nil { + log.Printf("vcs fs error: %s", err) + return rev, false + } + log.Printf("Rebuilding %s for %s", name, newRev) idx, err := buildAndOpenIndex( opt, dbpath, - vcsDir, + fs, nextIndexDir(dbpath), repo.Url, newRev) @@ -434,10 +440,15 @@ func newSearcher( refs.claim(ref) } + fs, err := wd.FileSystem(vcsDir) + if err != nil { + return nil, err + } + idx, err := buildAndOpenIndex( opt, dbpath, - vcsDir, + fs, idxDir, repo.Url, rev) diff --git a/vcs/bzr.go b/vcs/bzr.go index 1797635d..2563d178 100644 --- a/vcs/bzr.go +++ b/vcs/bzr.go @@ -81,3 +81,7 @@ func (g *BzrDriver) SpecialFiles() []string { func (g *BzrDriver) AutoGeneratedFiles(dir string) []string { return []string{} } + +func (g *BzrDriver) FileSystem(dir string) (FileSystem, error) { + return NewDirFilesystem(dir) +} diff --git a/vcs/dir_fs.go b/vcs/dir_fs.go new file mode 100644 index 00000000..31b7178d --- /dev/null +++ b/vcs/dir_fs.go @@ -0,0 +1,44 @@ +package vcs + +import ( + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" +) + +type dirFilesystem struct { + dir string +} + +func NewDirFilesystem(dir string) (FileSystem, error) { + // Resolve the symbolic link + if fi, err := os.Stat(dir); err == nil && fi.Mode()|os.ModeSymlink != 0 { + if s, err := os.Readlink(dir); err == nil { + dir = s + } + } + + return &dirFilesystem{dir: dir}, nil +} + +func (dir *dirFilesystem) Open(name string) (io.ReadCloser, error) { + if strings.HasPrefix(name, "/") { + return nil, fmt.Errorf("Expected relative path, got absolute: %s", name) + } + return os.Open(path.Join(dir.dir, name)) +} + +func (dir *dirFilesystem) Walk(fn filepath.WalkFunc) error { + return filepath.Walk(dir.dir, func(path string, info fs.FileInfo, err error) error { + rel, err := filepath.Rel(dir.dir, path) + if err != nil { + return err + } + + return fn(rel, info, err) + }) +} diff --git a/vcs/git.go b/vcs/git.go index b488d5cc..9b9bfb15 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -227,3 +227,7 @@ func (d *headBranchDetector) detectRef(dir string) string { return matches[1] } + +func (g *GitDriver) FileSystem(dir string) (FileSystem, error) { + return NewDirFilesystem(dir) +} diff --git a/vcs/hg.go b/vcs/hg.go index ac285836..8866a91b 100644 --- a/vcs/hg.go +++ b/vcs/hg.go @@ -83,3 +83,7 @@ func (g *MercurialDriver) SpecialFiles() []string { func (g *MercurialDriver) AutoGeneratedFiles(dir string) []string { return []string{} } + +func (g *MercurialDriver) FileSystem(dir string) (FileSystem, error) { + return NewDirFilesystem(dir) +} diff --git a/vcs/local.go b/vcs/local.go index 129edde3..97f7d3c2 100644 --- a/vcs/local.go +++ b/vcs/local.go @@ -105,3 +105,7 @@ func (g *LocalDriver) SpecialFiles() []string { func (g *LocalDriver) AutoGeneratedFiles(dir string) []string { return []string{} } + +func (g *LocalDriver) FileSystem(dir string) (FileSystem, error) { + return NewDirFilesystem(dir) +} diff --git a/vcs/svn.go b/vcs/svn.go index 0c8571a3..06b7fbdc 100644 --- a/vcs/svn.go +++ b/vcs/svn.go @@ -104,3 +104,7 @@ func (g *SVNDriver) SpecialFiles() []string { func (g *SVNDriver) AutoGeneratedFiles(dir string) []string { return []string{} } + +func (g *SVNDriver) FileSystem(dir string) (FileSystem, error) { + return NewDirFilesystem(dir) +} diff --git a/vcs/vcs.go b/vcs/vcs.go index 1e663274..1a5e404c 100644 --- a/vcs/vcs.go +++ b/vcs/vcs.go @@ -2,8 +2,10 @@ package vcs import ( "fmt" + "io" "log" "os" + "path/filepath" ) // A collection that maps vcs names to their underlying @@ -11,6 +13,14 @@ import ( // json config passed in to be parsed. var drivers = make(map[string]func(c []byte) (Driver, error)) +// A filesystem API abstraction to allow accessing vcs objects directly +// instead of duplicating them on disk +type FileSystem interface { + Open(name string) (io.ReadCloser, error) + + Walk(fn filepath.WalkFunc) error +} + // A "plugin" for each vcs that supports the very limited set of vcs // operations that hound needs. type Driver interface { @@ -29,6 +39,8 @@ type Driver interface { // Return a list of filenames that are marked as auto-generated. AutoGeneratedFiles(dir string) []string + + FileSystem(dir string) (FileSystem, error) } // An API to interact with a vcs working directory. This is From 69ff89574691391a3fbcb1bfe70eed53ba7b6ccf Mon Sep 17 00:00:00 2001 From: Tom van der Woerdt Date: Wed, 28 Aug 2024 14:10:45 +0200 Subject: [PATCH 3/5] git: get info from a 'bare' repo instead For git repositories you basically always pay the storage cost twice: once for the tree in .git, once for the actual files being checked out. Additionally, the tree in .git is compressed, so in many cases it can be smaller than the actual files being checked out. This commit updates Hound to optionally process git files from the actual git files instead of the checked out tree. For me, this saves a significant amount of disk space. Why does it matter? Because I run hound on a laptop instead of a datacenter, and space is limited. It does cost a few more CPU cycles, but that's fine by me. --- go.mod | 28 +++++++++++- go.sum | 109 ++++++++++++++++++++++++++++++++++++++++++++- index/index.go | 2 +- vcs/dir_fs.go | 2 +- vcs/git.go | 41 ++++++++++++----- vcs/git_fs.go | 117 +++++++++++++++++++++++++++++++++++++++++++++++++ vcs/vcs.go | 12 ++++- 7 files changed, 291 insertions(+), 20 deletions(-) create mode 100644 vcs/git_fs.go diff --git a/go.mod b/go.mod index 3d5151e6..87d79ebf 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,32 @@ module github.com/hound-search/hound -go 1.16 +go 1.18 require ( github.com/blang/semver/v4 v4.0.0 - golang.org/x/mod v0.10.0 + github.com/go-git/go-git/v5 v5.12.0 + golang.org/x/mod v0.12.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/tools v0.13.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index ab72a812..eaef6829 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +1,132 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/index/index.go b/index/index.go index e12ab3ec..cff82569 100644 --- a/index/index.go +++ b/index/index.go @@ -370,7 +370,7 @@ func indexAllFiles(opt *IndexOptions, dst string, fs vcs.FileSystem) error { } defer fileHandle.Close() - if err := fs.Walk(func(rel string, info os.FileInfo, err error) error { //nolint + if err := fs.Walk(func(rel string, info vcs.FileInfo, err error) error { //nolint name := info.Name() // Is this file considered "special", this means it's not even a part diff --git a/vcs/dir_fs.go b/vcs/dir_fs.go index 31b7178d..0e800968 100644 --- a/vcs/dir_fs.go +++ b/vcs/dir_fs.go @@ -32,7 +32,7 @@ func (dir *dirFilesystem) Open(name string) (io.ReadCloser, error) { return os.Open(path.Join(dir.dir, name)) } -func (dir *dirFilesystem) Walk(fn filepath.WalkFunc) error { +func (dir *dirFilesystem) Walk(fn FileSystemWalkFunc) error { return filepath.Walk(dir.dir, func(path string, info fs.FileInfo, err error) error { rel, err := filepath.Rel(dir.dir, path) if err != nil { diff --git a/vcs/git.go b/vcs/git.go index 9b9bfb15..7049b0f7 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -24,6 +24,7 @@ func init() { type GitDriver struct { DetectRef bool `json:"detect-ref"` Ref string `json:"ref"` + Bare bool `json:"bare"` refDetetector refDetetector } @@ -49,10 +50,15 @@ func newGit(b []byte) (Driver, error) { } func (g *GitDriver) HeadRev(dir string) (string, error) { + targetRef := "HEAD" + if g.Bare { + targetRef = fmt.Sprintf("origin/%s", g.targetRef(dir)) + } + cmd := exec.Command( "git", "rev-parse", - "HEAD") + targetRef) cmd.Dir = dir r, err := cmd.StdoutPipe() if err != nil { @@ -102,12 +108,15 @@ func (g *GitDriver) Pull(dir string) (string, error) { return "", err } - if _, err := run("git reset", dir, - "git", - "reset", - "--hard", - fmt.Sprintf("origin/%s", targetRef)); err != nil { - return "", err + if !g.Bare { + // XXX(tvdw) Check if this works without fetch? + if _, err := run("git reset", dir, + "git", + "reset", + "--hard", + fmt.Sprintf("origin/%s", targetRef)); err != nil { + return "", err + } } return g.HeadRev(dir) @@ -130,12 +139,16 @@ func (g *GitDriver) targetRef(dir string) string { func (g *GitDriver) Clone(dir, url string) (string, error) { par, rep := filepath.Split(dir) - cmd := exec.Command( - "git", + + cmdArgs := []string{ "clone", "--depth", "1", - url, - rep) + } + if g.Bare { + cmdArgs = append(cmdArgs, "--bare") + } + cmdArgs = append(cmdArgs, url, rep) + cmd := exec.Command("git", cmdArgs...) cmd.Dir = par out, err := cmd.CombinedOutput() if err != nil { @@ -229,5 +242,9 @@ func (d *headBranchDetector) detectRef(dir string) string { } func (g *GitDriver) FileSystem(dir string) (FileSystem, error) { - return NewDirFilesystem(dir) + if g.Bare { + return NewGitFilesystem(dir, fmt.Sprintf("origin/%s", g.targetRef(dir))) + } else { + return NewDirFilesystem(dir) + } } diff --git a/vcs/git_fs.go b/vcs/git_fs.go new file mode 100644 index 00000000..4808757d --- /dev/null +++ b/vcs/git_fs.go @@ -0,0 +1,117 @@ +package vcs + +import ( + "io" + "io/fs" + "path" + "slices" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +type gitFilesystem struct { + repo *gogit.Repository + root *object.Tree +} + +type gitFileinfo struct { + raw *object.File +} + +type gitDirinfo struct { + name string +} + +func NewGitFilesystem(dir, ref string) (FileSystem, error) { + repo, err := gogit.PlainOpen(dir) + if err != nil { + return nil, err + } + + rev, err := repo.ResolveRevision(plumbing.Revision(ref)) + if err != nil { + return nil, err + } + + commit, err := repo.CommitObject(*rev) + if err != nil { + return nil, err + } + + root, err := commit.Tree() + if err != nil { + return nil, err + } + + return &gitFilesystem{ + repo: repo, + root: root, + }, nil +} + +func (fs *gitFilesystem) Open(name string) (io.ReadCloser, error) { + file, err := fs.root.File(name) + if err != nil { + return nil, err + } + return file.Reader() +} + +func (fs *gitFilesystem) Walk(fn FileSystemWalkFunc) error { + seenDirs := make(map[string]interface{}) + + return fs.root.Files().ForEach(func(f *object.File) error { + n := f.Name + var createDirs []string + for n != "" { + n = path.Dir(n) + if _, ok := seenDirs[n]; ok { + break + } + seenDirs[n] = true + createDirs = append(createDirs, n) + } + if len(createDirs) > 0 { + slices.Reverse(createDirs) + for _, createDir := range createDirs { + err := fn(createDir, &gitDirinfo{createDir}, nil) + if err != nil { + return err + } + } + } + + fi := &gitFileinfo{f} + if fi.IsDir() { + seenDirs[f.Name] = true + } + + return fn(f.Name, fi, nil) + }) +} + +func (fi *gitFileinfo) Name() string { + return path.Base(fi.raw.Name) +} + +func (fi *gitFileinfo) IsDir() bool { + return fi.raw.Mode == filemode.Dir +} + +func (fi *gitFileinfo) Mode() fs.FileMode { + mode, _ := fi.raw.Mode.ToOSFileMode() + return mode +} + +func (di *gitDirinfo) Name() string { + return path.Base(di.name) +} +func (di *gitDirinfo) IsDir() bool { + return true +} +func (di *gitDirinfo) Mode() fs.FileMode { + return fs.FileMode(0o755) +} diff --git a/vcs/vcs.go b/vcs/vcs.go index 1a5e404c..6f0ef3c9 100644 --- a/vcs/vcs.go +++ b/vcs/vcs.go @@ -3,9 +3,9 @@ package vcs import ( "fmt" "io" + "io/fs" "log" "os" - "path/filepath" ) // A collection that maps vcs names to their underlying @@ -15,10 +15,18 @@ var drivers = make(map[string]func(c []byte) (Driver, error)) // A filesystem API abstraction to allow accessing vcs objects directly // instead of duplicating them on disk +type FileInfo interface { + Name() string + IsDir() bool + Mode() fs.FileMode +} + +type FileSystemWalkFunc func(path string, info FileInfo, err error) error + type FileSystem interface { Open(name string) (io.ReadCloser, error) - Walk(fn filepath.WalkFunc) error + Walk(fn FileSystemWalkFunc) error } // A "plugin" for each vcs that supports the very limited set of vcs From 8172ac98bed5c2a95fb12d249dd2a156a4ccc5bd Mon Sep 17 00:00:00 2001 From: Tom van der Woerdt Date: Wed, 28 Aug 2024 15:49:12 +0200 Subject: [PATCH 4/5] git: implement git on top of go-git Code is simpler, faster, and we can get rid of ref detection (could previously already have done that, just need to pull HEAD). --- vcs/git.go | 171 +++++++++++------------------------------------- vcs/git_test.go | 85 ------------------------ vcs/vcs.go | 3 - 3 files changed, 37 insertions(+), 222 deletions(-) delete mode 100644 vcs/git_test.go diff --git a/vcs/git.go b/vcs/git.go index 7049b0f7..05604cd8 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -4,35 +4,23 @@ import ( "bytes" "encoding/json" "fmt" - "io" "log" "os/exec" - "path/filepath" - "regexp" - "strings" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" ) -const defaultRef = "master" +const indexRef = "localindex" const autoGeneratedAttribute = "linguist-generated" -var headBranchRegexp = regexp.MustCompile(`HEAD branch: (?P.+)`) - func init() { Register(newGit, "git") } type GitDriver struct { - DetectRef bool `json:"detect-ref"` - Ref string `json:"ref"` - Bare bool `json:"bare"` - refDetetector refDetetector -} - -type refDetetector interface { - detectRef(dir string) string -} - -type headBranchDetector struct { + Ref string `json:"ref"` + Bare bool `json:"bare"` } func newGit(b []byte) (Driver, error) { @@ -44,115 +32,56 @@ func newGit(b []byte) (Driver, error) { } } - d.refDetetector = &headBranchDetector{} - return &d, nil } -func (g *GitDriver) HeadRev(dir string) (string, error) { - targetRef := "HEAD" - if g.Bare { - targetRef = fmt.Sprintf("origin/%s", g.targetRef(dir)) - } - - cmd := exec.Command( - "git", - "rev-parse", - targetRef) - cmd.Dir = dir - r, err := cmd.StdoutPipe() +func (g *GitDriver) Pull(dir string) (string, error) { + repo, err := gogit.PlainOpen(dir) if err != nil { return "", err } - defer r.Close() - if err := cmd.Start(); err != nil { - return "", err + fetchTarget := "HEAD" + if g.Ref != "" { + fetchTarget = g.Ref } - var buf bytes.Buffer - - if _, err := io.Copy(&buf, r); err != nil { + if err = repo.Fetch(&gogit.FetchOptions{ + Depth: 1, + Prune: true, + Tags: gogit.NoTags, + RefSpecs: []config.RefSpec{ + config.RefSpec(fmt.Sprintf("+%s:refs/heads/%s", fetchTarget, indexRef)), + }, + }); err != nil { return "", err } - return strings.TrimSpace(buf.String()), cmd.Wait() -} - -func run(desc, dir, cmd string, args ...string) (string, error) { - c := exec.Command(cmd, args...) - c.Dir = dir - out, err := c.CombinedOutput() + newHead, err := repo.ResolveRevision(indexRef) if err != nil { - log.Printf( - "Failed to %s %v at %q, see output below\n%s: %+v\nContinuing...", - desc, - c.Args, c.Dir, - out, err) - } - - return string(out), nil -} - -func (g *GitDriver) Pull(dir string) (string, error) { - targetRef := g.targetRef(dir) - - if _, err := run("git fetch", dir, - "git", - "fetch", - "--prune", - "--no-tags", - "--depth", "1", - "origin", - fmt.Sprintf("+%s:remotes/origin/%s", targetRef, targetRef)); err != nil { return "", err } if !g.Bare { - // XXX(tvdw) Check if this works without fetch? - if _, err := run("git reset", dir, - "git", - "reset", - "--hard", - fmt.Sprintf("origin/%s", targetRef)); err != nil { + worktree, err := repo.Worktree() + if err != nil { return "", err } + worktree.Reset(&gogit.ResetOptions{ + Mode: gogit.HardReset, + Commit: *newHead, + }) } - return g.HeadRev(dir) -} - -func (g *GitDriver) targetRef(dir string) string { - var targetRef string - if g.Ref != "" { - targetRef = g.Ref - } else if g.DetectRef { - targetRef = g.refDetetector.detectRef(dir) - } - - if targetRef == "" { - targetRef = defaultRef - } - - return targetRef + return newHead.String(), nil } func (g *GitDriver) Clone(dir, url string) (string, error) { - par, rep := filepath.Split(dir) - - cmdArgs := []string{ - "clone", - "--depth", "1", - } - if g.Bare { - cmdArgs = append(cmdArgs, "--bare") - } - cmdArgs = append(cmdArgs, url, rep) - cmd := exec.Command("git", cmdArgs...) - cmd.Dir = par - out, err := cmd.CombinedOutput() + _, err := gogit.PlainClone(dir, g.Bare, &gogit.CloneOptions{ + Depth: 1, + URL: url, + }) if err != nil { - log.Printf("Failed to clone %s, see output below\n%sContinuing...", url, out) return "", err } @@ -168,6 +97,11 @@ func (g *GitDriver) SpecialFiles() []string { func (g *GitDriver) AutoGeneratedFiles(dir string) []string { var files []string + // XXX(tvdw): broken under Bare + if g.Bare { + return nil + } + filesCmd := exec.Command("git", "ls-files", "-z") filesCmd.Dir = dir pipe, err := filesCmd.StdoutPipe() @@ -210,40 +144,9 @@ func (g *GitDriver) AutoGeneratedFiles(dir string) []string { return files } -func (d *headBranchDetector) detectRef(dir string) string { - output, err := run("git show remote info", dir, - "git", - "remote", - "show", - "origin", - ) - - if err != nil { - log.Printf( - "error occured when fetching info to determine target ref in %s: %s. Will fall back to default ref %s", - dir, - err, - defaultRef, - ) - return "" - } - - matches := headBranchRegexp.FindStringSubmatch(output) - if len(matches) != 2 { - log.Printf( - "could not determine target ref in %s. Will fall back to default ref %s", - dir, - defaultRef, - ) - return "" - } - - return matches[1] -} - func (g *GitDriver) FileSystem(dir string) (FileSystem, error) { if g.Bare { - return NewGitFilesystem(dir, fmt.Sprintf("origin/%s", g.targetRef(dir))) + return NewGitFilesystem(dir, indexRef) } else { return NewDirFilesystem(dir) } diff --git a/vcs/git_test.go b/vcs/git_test.go deleted file mode 100644 index 737b324b..00000000 --- a/vcs/git_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package vcs - -import ( - "fmt" - "testing" -) - -type testRefDetector struct { - result string -} - -func (d *testRefDetector) detectRef(dir string) string { - return d.result -} - -func TestTargetRef(t *testing.T) { - testCases := []struct { - explicitRef string - detectRefEnabled bool - detectRefResult string - expectedResult string - }{ - { - explicitRef: "", - detectRefEnabled: true, - detectRefResult: "detected-ref", - expectedResult: "detected-ref", - }, - { - explicitRef: "", - detectRefEnabled: true, - detectRefResult: "", - expectedResult: defaultRef, - }, - { - explicitRef: "explicit-ref", - detectRefEnabled: true, - detectRefResult: "detected-ref", - expectedResult: "explicit-ref", - }, - { - explicitRef: "explicit-ref", - detectRefEnabled: true, - detectRefResult: "", - expectedResult: "explicit-ref", - }, - { - explicitRef: "explicit-ref", - detectRefEnabled: false, - detectRefResult: "foo", - expectedResult: "explicit-ref", - }, - { - explicitRef: "", - detectRefEnabled: false, - detectRefResult: "", - expectedResult: defaultRef, - }, - { - explicitRef: "explicit-ref", - detectRefEnabled: false, - detectRefResult: "", - expectedResult: "explicit-ref", - }, - { - explicitRef: "explicit-ref", - detectRefEnabled: false, - detectRefResult: "detected-ref", - expectedResult: "explicit-ref", - }, - } - for idx, testCase := range testCases { - t.Run(fmt.Sprintf("test case %d", idx), func(t *testing.T) { - driver := &GitDriver{ - Ref: testCase.explicitRef, - DetectRef: testCase.detectRefEnabled, - refDetetector: &testRefDetector{result: testCase.detectRefResult}, - } - actualResult := driver.targetRef("dir") - if actualResult != testCase.expectedResult { - t.Errorf("expected target ref: %q, got: %q", testCase.expectedResult, actualResult) - } - }) - } -} diff --git a/vcs/vcs.go b/vcs/vcs.go index 6f0ef3c9..17d72abb 100644 --- a/vcs/vcs.go +++ b/vcs/vcs.go @@ -39,9 +39,6 @@ type Driver interface { // Pull new changes from the server and update the working directory. Pull(dir string) (string, error) - // Return the revision at the head of the vcs directory. - HeadRev(dir string) (string, error) - // Return a list of special filenames that should not be indexed. SpecialFiles() []string From bc476540118d4807db0af3f42d6d2db5f377337a Mon Sep 17 00:00:00 2001 From: Tom van der Woerdt Date: Wed, 28 Aug 2024 16:34:03 +0200 Subject: [PATCH 5/5] Some cleanups on how directories are created from git git doesn't really have a concept of a directory, only files, so we have to figure them out ourselves. My previous code for that was a bit silly, this one still is. --- vcs/git.go | 10 +++++----- vcs/git_fs.go | 36 ++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/vcs/git.go b/vcs/git.go index 05604cd8..cc7eb1e8 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -38,7 +38,7 @@ func newGit(b []byte) (Driver, error) { func (g *GitDriver) Pull(dir string) (string, error) { repo, err := gogit.PlainOpen(dir) if err != nil { - return "", err + return "", fmt.Errorf("failed to open repository: %w", err) } fetchTarget := "HEAD" @@ -54,18 +54,18 @@ func (g *GitDriver) Pull(dir string) (string, error) { config.RefSpec(fmt.Sprintf("+%s:refs/heads/%s", fetchTarget, indexRef)), }, }); err != nil { - return "", err + return "", fmt.Errorf("failed to pull: %w", err) } newHead, err := repo.ResolveRevision(indexRef) if err != nil { - return "", err + return "", fmt.Errorf("failed to resolve revision: %w", err) } if !g.Bare { worktree, err := repo.Worktree() if err != nil { - return "", err + return "", fmt.Errorf("failed to reset worktree: %w", err) } worktree.Reset(&gogit.ResetOptions{ Mode: gogit.HardReset, @@ -82,7 +82,7 @@ func (g *GitDriver) Clone(dir, url string) (string, error) { URL: url, }) if err != nil { - return "", err + return "", fmt.Errorf("failed to clone %s: %w", url, err) } return g.Pull(dir) diff --git a/vcs/git_fs.go b/vcs/git_fs.go index 4808757d..9ad691aa 100644 --- a/vcs/git_fs.go +++ b/vcs/git_fs.go @@ -61,35 +61,31 @@ func (fs *gitFilesystem) Open(name string) (io.ReadCloser, error) { } func (fs *gitFilesystem) Walk(fn FileSystemWalkFunc) error { - seenDirs := make(map[string]interface{}) + seenDirs := make(map[string]bool) return fs.root.Files().ForEach(func(f *object.File) error { n := f.Name var createDirs []string - for n != "" { + if f.Mode != filemode.Dir { n = path.Dir(n) - if _, ok := seenDirs[n]; ok { - break - } + } + for n != "" && !seenDirs[n] { seenDirs[n] = true createDirs = append(createDirs, n) + n = path.Dir(n) } - if len(createDirs) > 0 { - slices.Reverse(createDirs) - for _, createDir := range createDirs { - err := fn(createDir, &gitDirinfo{createDir}, nil) - if err != nil { - return err - } + slices.Reverse(createDirs) + for _, createDir := range createDirs { + if err := fn(createDir, &gitDirinfo{n}, nil); err != nil { + return err } } - fi := &gitFileinfo{f} - if fi.IsDir() { - seenDirs[f.Name] = true + if f.Mode != filemode.Dir { + return fn(f.Name, &gitFileinfo{f}, nil) + } else { + return nil } - - return fn(f.Name, fi, nil) }) } @@ -98,7 +94,7 @@ func (fi *gitFileinfo) Name() string { } func (fi *gitFileinfo) IsDir() bool { - return fi.raw.Mode == filemode.Dir + return fi.Mode().IsDir() } func (fi *gitFileinfo) Mode() fs.FileMode { @@ -110,8 +106,8 @@ func (di *gitDirinfo) Name() string { return path.Base(di.name) } func (di *gitDirinfo) IsDir() bool { - return true + return di.Mode().IsDir() } func (di *gitDirinfo) Mode() fs.FileMode { - return fs.FileMode(0o755) + return fs.FileMode(0o755 | fs.ModeDir) }