Add basic repository lfs management

models/lfs.go

@@ -8,6 +8,8 @@ import (
 	"io"
 
 	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/builder"
 )
 
 // LFSMetaObject stores metadata for LFS tracked files.
@@ -106,19 +108,91 @@ func (repo *Repository) GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error
 
 // RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
 // It may return ErrLFSObjectNotExist or a database error.
-func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) error {
+func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) (int64, error) {
 	if len(oid) == 0 {
-		return ErrLFSObjectNotExist
+		return 0, ErrLFSObjectNotExist
 	}
 
 	sess := x.NewSession()
 	defer sess.Close()
 	if err := sess.Begin(); err != nil {
-		return err
+		return -1, err
 	}
 
 	m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID}
 	if _, err := sess.Delete(m); err != nil {
+		return -1, err
+	}
+
+	count, err := sess.Count(&LFSMetaObject{Oid: oid})
+	if err != nil {
+		return count, err
+	}
+
+	return count, sess.Commit()
+}
+
+// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository
+func (repo *Repository) GetLFSMetaObjects(page, pageSize int) ([]*LFSMetaObject, error) {
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if page >= 0 && pageSize > 0 {
+		start := 0
+		if page > 0 {
+			start = (page - 1) * pageSize
+		}
+		sess.Limit(pageSize, start)
+	}
+	lfsObjects := make([]*LFSMetaObject, 0, pageSize)
+	return lfsObjects, sess.Find(&lfsObjects, &LFSMetaObject{RepositoryID: repo.ID})
+}
+
+// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository
+func (repo *Repository) CountLFSMetaObjects() (int64, error) {
+	return x.Count(&LFSMetaObject{RepositoryID: repo.ID})
+}
+
+// LFSObjectAccessible checks if a provided Oid is accessible to the user
+func LFSObjectAccessible(user *User, oid string) (bool, error) {
+	if user.IsAdmin {
+		count, err := x.Count(&LFSMetaObject{Oid: oid})
+		return (count > 0), err
+	}
+	cond := accessibleRepositoryCondition(user.ID)
+	count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid})
+	return (count > 0), err
+}
+
+// LFSAutoAssociate auto associates accessible LFSMetaObjects
+func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	oids := make([]interface{}, len(metas))
+	oidMap := make(map[string]*LFSMetaObject, len(metas))
+	for i, meta := range metas {
+		oids[i] = meta.Oid
+		oidMap[meta.Oid] = meta
+	}
+
+	cond := builder.NewCond()
+	if !user.IsAdmin {
+		cond = builder.In("`lfs_meta_object`.repository_id",
+			builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user.ID)))
+	}
+	newMetas := make([]*LFSMetaObject, 0, len(metas))
+	if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil {
+		return err
+	}
+	for i := range newMetas {
+		newMetas[i].Size = oidMap[newMetas[i].Oid].Size
+		newMetas[i].RepositoryID = repoID
+	}
+	if _, err := sess.InsertMulti(newMetas); err != nil {
 		return err
 	}
 

models/repo_list.go

@@ -176,28 +176,7 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
 	if opts.Private {
 		if !opts.UserIsAdmin && opts.UserID != 0 && opts.UserID != opts.OwnerID {
 			// OK we're in the context of a User
-			// We should be Either
-			cond = cond.And(builder.Or(
-				// 1. Be able to see all non-private repositories that either:
-				cond.And(
-					builder.Eq{"is_private": false},
-					builder.Or(
-						//   A. Aren't in organisations  __OR__
-						builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})),
-						//   B. Isn't a private organisation. (Limited is OK because we're logged in)
-						builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))),
-				),
-				// 2. Be able to see all repositories that we have access to
-				builder.In("id", builder.Select("repo_id").
-					From("`access`").
-					Where(builder.And(
-						builder.Eq{"user_id": opts.UserID},
-						builder.Gt{"mode": int(AccessModeNone)}))),
-				// 3. Be able to see all repositories that we are in a team
-				builder.In("id", builder.Select("`team_repo`.repo_id").
-					From("team_repo").
-					Where(builder.Eq{"`team_user`.uid": opts.UserID}).
-					Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id"))))
+			cond = cond.And(accessibleRepositoryCondition(opts.UserID))
 		}
 	} else {
 		// Not looking at private organisations
@@ -316,6 +295,31 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
 	return repos, count, nil
 }
 
+// accessibleRepositoryCondition takes a user a returns a condition for checking if a repository is accessible
+func accessibleRepositoryCondition(userID int64) builder.Cond {
+	return builder.Or(
+		// 1. Be able to see all non-private repositories that either:
+		builder.And(
+			builder.Eq{"`repository`.is_private": false},
+			builder.Or(
+				//   A. Aren't in organisations  __OR__
+				builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})),
+				//   B. Isn't a private organisation. (Limited is OK because we're logged in)
+				builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))),
+		),
+		// 2. Be able to see all repositories that we have access to
+		builder.In("`repository`.id", builder.Select("repo_id").
+			From("`access`").
+			Where(builder.And(
+				builder.Eq{"user_id": userID},
+				builder.Gt{"mode": int(AccessModeNone)}))),
+		// 3. Be able to see all repositories that we are in a team
+		builder.In("`repository`.id", builder.Select("`team_repo`.repo_id").
+			From("team_repo").
+			Where(builder.Eq{"`team_user`.uid": userID}).
+			Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id")))
+}
+
 // SearchRepositoryByName takes keyword and part of repository name to search,
 // it returns results in given range and number of total results.
 func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, error) {

modules/git/pipeline/catfile.go

@@ -0,0 +1,94 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+)
+
+// CatFileBatchCheck runs cat-file with --batch-check
+func CatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
+	defer wg.Done()
+	defer shasToCheckReader.Close()
+	defer catFileCheckWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("cat-file", "--batch-check")
+	if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil {
+		_ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
+	}
+}
+
+// CatFileBatchCheckAllObjects runs cat-file with --batch-check --batch-all
+func CatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) {
+	defer wg.Done()
+	defer catFileCheckWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects")
+	if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil {
+		log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+		err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+		_ = catFileCheckWriter.CloseWithError(err)
+		errChan <- err
+	}
+}
+
+// CatFileBatch runs cat-file --batch
+func CatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
+	defer wg.Done()
+	defer shasToBatchReader.Close()
+	defer catFileBatchWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil {
+		_ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
+	}
+}
+
+// BlobsLessThan1024FromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <1024 in size
+func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) {
+	defer wg.Done()
+	defer catFileCheckReader.Close()
+	scanner := bufio.NewScanner(catFileCheckReader)
+	defer func() {
+		_ = shasToBatchWriter.CloseWithError(scanner.Err())
+	}()
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		fields := strings.Split(line, " ")
+		if len(fields) < 3 || fields[1] != "blob" {
+			continue
+		}
+		size, _ := strconv.Atoi(fields[2])
+		if size > 1024 {
+			continue
+		}
+		toWrite := []byte(fields[0] + "\n")
+		for len(toWrite) > 0 {
+			n, err := shasToBatchWriter.Write(toWrite)
+			if err != nil {
+				_ = catFileCheckReader.CloseWithError(err)
+				break
+			}
+			toWrite = toWrite[n:]
+		}
+	}
+}

modules/git/pipeline/namerev.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+)
+
+// NameRevStdin runs name-rev --stdin
+func NameRevStdin(shasToNameReader *io.PipeReader, nameRevStdinWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
+	defer wg.Done()
+	defer shasToNameReader.Close()
+	defer nameRevStdinWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").RunInDirFullPipeline(tmpBasePath, nameRevStdinWriter, stderr, shasToNameReader); err != nil {
+		_ = shasToNameReader.CloseWithError(fmt.Errorf("git name-rev [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
+	}
+}

modules/git/pipeline/revlist.go

@@ -0,0 +1,75 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+)
+
+// RevListAllObjects runs rev-list --objects --all and writes to a pipewriter
+func RevListAllObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) {
+	defer wg.Done()
+	defer revListWriter.Close()
+
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("rev-list", "--objects", "--all")
+	if err := cmd.RunInDirPipeline(basePath, revListWriter, stderr); err != nil {
+		log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
+		err = fmt.Errorf("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
+		_ = revListWriter.CloseWithError(err)
+		errChan <- err
+	}
+}
+
+// RevListObjects run rev-list --objects from headSHA to baseSHA
+func RevListObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) {
+	defer wg.Done()
+	defer revListWriter.Close()
+	stderr := new(bytes.Buffer)
+	var errbuf strings.Builder
+	cmd := git.NewCommand("rev-list", "--objects", headSHA, "--not", baseSHA)
+	if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil {
+		log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+		errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
+	}
+}
+
+// BlobsFromRevListObjects reads a RevListAllObjects and only selects blobs
+func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) {
+	defer wg.Done()
+	defer revListReader.Close()
+	scanner := bufio.NewScanner(revListReader)
+	defer func() {
+		_ = shasToCheckWriter.CloseWithError(scanner.Err())
+	}()
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		fields := strings.Split(line, " ")
+		if len(fields) < 2 || len(fields[1]) == 0 {
+			continue
+		}
+		toWrite := []byte(fields[0] + "\n")
+		for len(toWrite) > 0 {
+			n, err := shasToCheckWriter.Write(toWrite)
+			if err != nil {
+				_ = revListReader.CloseWithError(err)
+				break
+			}
+			toWrite = toWrite[n:]
+		}
+	}
+}

modules/git/repo.go

@@ -107,6 +107,11 @@ func OpenRepository(repoPath string) (*Repository, error) {
 	}, nil
 }
 
+// GoGitRepo gets the go-git repo representation
+func (repo *Repository) GoGitRepo() *gogit.Repository {
+	return repo.gogitRepo
+}
+
 // IsEmpty Check if repository is empty.
 func (repo *Repository) IsEmpty() (bool, error) {
 	var errbuf strings.Builder

modules/lfs/server.go

@@ -330,7 +330,7 @@ func PutHandler(ctx *context.Context) {
 	if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil {
 		ctx.Resp.WriteHeader(500)
 		fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
-		if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
+		if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
 			log.Error("RemoveLFSMetaObjectByOid: %v", err)
 		}
 		return

modules/repofiles/update.go

@@ -383,7 +383,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
 		contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
 		if !contentStore.Exists(lfsMetaObject) {
 			if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
-				if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
+				if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
 					return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
 				}
 				return nil, err