Use global lock instead of NewExclusivePool to allow distributed lock between multiple Gitea instances

custom/conf/app.example.ini

@@ -2713,3 +2713,9 @@ LEVEL = Info
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; storage type
 ;STORAGE_TYPE = local
+
+;[global_lock]
+;; Lock service type, could be memory or redis
+;SERVICE_TYPE = memory
+;; Ignored for the "memory" type. For "redis" use something like `redis://127.0.0.1:6379/0`
+;SERVICE_CONN_STR =

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/go-ldap/ldap/v3 v3.4.6
+	github.com/go-redsync/redsync/v4 v4.13.0
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-swagger/go-swagger v0.31.0
 	github.com/go-testfixtures/testfixtures/v3 v3.11.0
@@ -218,7 +219,9 @@ require (
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect

go.sum

@@ -342,6 +342,14 @@ github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ
 github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
 github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
 github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
+github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
+github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
+github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
+github.com/go-redsync/redsync/v4 v4.13.0 h1:49X6GJfnbLGaIpBBREM/zA4uIMDXKAh1NDkvQ1EkZKA=
+github.com/go-redsync/redsync/v4 v4.13.0/go.mod h1:HMW4Q224GZQz6x1Xc7040Yfgacukdzu7ifTDAKiyErQ=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
@@ -397,6 +405,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -449,10 +459,15 @@ github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE=
 github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -674,6 +689,8 @@ github.com/quasoft/websspi v1.1.2/go.mod h1:HmVdl939dQ0WIXZhyik+ARdI03M6bQzaSEKc
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.6.0 h1:NLck+Rab3AOTHw21CGRpvQpgTrAU4sgdCswqGtlhGRA=
 github.com/redis/go-redis/v9 v9.6.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo=
+github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rhysd/actionlint v1.7.1 h1:WJaDzyT1StBWVKGSsZPYnbV0HF9Y9/vD6KFdZQL42qE=
@@ -765,6 +782,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM=
+github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=

modules/globallock/lock.go

@@ -0,0 +1,137 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package globallock
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/modules/nosql"
+	"code.gitea.io/gitea/modules/setting"
+
+	redsync "github.com/go-redsync/redsync/v4"
+	goredis "github.com/go-redsync/redsync/v4/redis/goredis/v9"
+)
+
+type Locker interface {
+	Lock() error            // lock the resource and block until it is unlocked by the holder
+	TryLock() (bool, error) // try to lock the resource and return immediately, first return value indicates if the lock was successful
+	Unlock() (bool, error)  // only lock with no error and TryLock returned true with no error can be unlocked
+}
+
+type LockService interface {
+	GetLocker(name string) Locker // create or get a locker by name, RemoveLocker should be called after the locker is no longer needed
+	RemoveLocker(name string)     // remove a locker by name from the pool. This should be invoked affect locker is no longer needed, i.e. a pull request merged or closed
+}
+
+type memoryLock struct {
+	mutex sync.Mutex
+}
+
+func (r *memoryLock) Lock() error {
+	r.mutex.Lock()
+	return nil
+}
+
+func (r *memoryLock) TryLock() (bool, error) {
+	return r.mutex.TryLock(), nil
+}
+
+func (r *memoryLock) Unlock() (bool, error) {
+	r.mutex.Unlock()
+	return true, nil
+}
+
+var _ Locker = &memoryLock{}
+
+type memoryLockService struct {
+	syncMap sync.Map
+}
+
+var _ LockService = &memoryLockService{}
+
+func newMemoryLockService() *memoryLockService {
+	return &memoryLockService{
+		syncMap: sync.Map{},
+	}
+}
+
+func (l *memoryLockService) GetLocker(name string) Locker {
+	v, _ := l.syncMap.LoadOrStore(name, &memoryLock{})
+	return v.(*memoryLock)
+}
+
+func (l *memoryLockService) RemoveLocker(name string) {
+	l.syncMap.Delete(name)
+}
+
+type redisLockService struct {
+	rs *redsync.Redsync
+}
+
+var _ LockService = &redisLockService{}
+
+func newRedisLockService(connection string) *redisLockService {
+	client := nosql.GetManager().GetRedisClient(connection)
+
+	pool := goredis.NewPool(client)
+
+	// Create an instance of redisync to be used to obtain a mutual exclusion
+	// lock.
+	rs := redsync.New(pool)
+
+	return &redisLockService{
+		rs: rs,
+	}
+}
+
+type redisLock struct {
+	mutex *redsync.Mutex
+}
+
+func (r *redisLockService) GetLocker(name string) Locker {
+	return &redisLock{mutex: r.rs.NewMutex(name)}
+}
+
+func (r *redisLockService) RemoveLocker(name string) {
+	// Do nothing
+}
+
+func (r *redisLock) Lock() error {
+	return r.mutex.Lock()
+}
+
+func (r *redisLock) TryLock() (bool, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+	defer cancel()
+	if err := r.mutex.LockContext(ctx); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func (r *redisLock) Unlock() (bool, error) {
+	return r.mutex.Unlock()
+}
+
+var (
+	syncOnce    sync.Once
+	lockService LockService
+)
+
+func getLockService() LockService {
+	syncOnce.Do(func() {
+		if setting.GlobalLock.ServiceType == "redis" {
+			lockService = newRedisLockService(setting.GlobalLock.ServiceConnStr)
+		} else {
+			lockService = newMemoryLockService()
+		}
+	})
+	return lockService
+}
+
+func GetLocker(name string) Locker {
+	return getLockService().GetLocker(name)
+}

modules/globallock/lock_test.go

@@ -0,0 +1,65 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package globallock
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Lock(t *testing.T) {
+	locker1 := GetLocker("test2")
+	assert.NoError(t, locker1.Lock())
+	unlocked, err := locker1.Unlock()
+	assert.NoError(t, err)
+	assert.True(t, unlocked)
+
+	locker2 := GetLocker("test2")
+	assert.NoError(t, locker2.Lock())
+
+	locked1, err1 := locker2.TryLock()
+	assert.NoError(t, err1)
+	assert.False(t, locked1)
+
+	locker2.Unlock()
+
+	locked2, err2 := locker2.TryLock()
+	assert.NoError(t, err2)
+	assert.True(t, locked2)
+
+	locker2.Unlock()
+}
+
+func Test_Lock_Redis(t *testing.T) {
+	if os.Getenv("CI") == "" {
+		t.Skip("Skip test for local development")
+	}
+
+	lockService = newRedisLockService("redis://redis")
+
+	redisPool :=
+	locker1 := GetLocker("test1")
+	assert.NoError(t, locker1.Lock())
+	unlocked, err := locker1.Unlock()
+	assert.NoError(t, err)
+	assert.True(t, unlocked)
+
+	locker2 := GetLocker("test1")
+	assert.NoError(t, locker2.Lock())
+
+	locked1, err1 := locker2.TryLock()
+	assert.NoError(t, err1)
+	assert.False(t, locked1)
+
+	locker2.Unlock()
+
+	locked2, err2 := locker2.TryLock()
+	assert.NoError(t, err2)
+	assert.True(t, locked2)
+
+	locker2.Unlock()
+
+	redisPool.Close()
+}

modules/setting/gloabl_lock.go

@@ -0,0 +1,37 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package setting
+
+import (
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/nosql"
+)
+
+// GlobalLock represents configuration of global lock
+var GlobalLock = struct {
+	ServiceType    string
+	ServiceConnStr string
+}{
+	ServiceType: "memory",
+}
+
+func loadGlobalLockFrom(rootCfg ConfigProvider) {
+	sec := rootCfg.Section("global_lock")
+	GlobalLock.ServiceType = sec.Key("SERVICE_TYPE").MustString("memory")
+	switch GlobalLock.ServiceType {
+	case "memory":
+	case "redis":
+		connStr := sec.Key("SERVICE_CONN_STR").String()
+		if connStr == "" {
+			log.Fatal("SERVICE_CONN_STR is empty for redis")
+		}
+		u := nosql.ToRedisURI(connStr)
+		if u == nil {
+			log.Fatal("SERVICE_CONN_STR %s is not a valid redis connection string", connStr)
+		}
+		GlobalLock.ServiceConnStr = connStr
+	default:
+		log.Fatal("Unknown sync lock service type: %s", GlobalLock.ServiceType)
+	}
+}

modules/setting/global_lock_test.go

@@ -0,0 +1,35 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package setting
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadGlobalLockConfig(t *testing.T) {
+	t.Run("DefaultGlobalLockConfig", func(t *testing.T) {
+		iniStr := ``
+		cfg, err := NewConfigProviderFromData(iniStr)
+		assert.NoError(t, err)
+
+		loadGlobalLockFrom(cfg)
+		assert.EqualValues(t, "memory", GlobalLock.ServiceType)
+	})
+
+	t.Run("RedisGlobalLockConfig", func(t *testing.T) {
+		iniStr := `
+[global_lock]
+SERVICE_TYPE = redis
+SERVICE_CONN_STR = addrs=127.0.0.1:6379 db=0
+`
+		cfg, err := NewConfigProviderFromData(iniStr)
+		assert.NoError(t, err)
+
+		loadGlobalLockFrom(cfg)
+		assert.EqualValues(t, "redis", GlobalLock.ServiceType)
+		assert.EqualValues(t, "addrs=127.0.0.1:6379 db=0", GlobalLock.ServiceConnStr)
+	})
+}

modules/setting/setting.go

@@ -147,6 +147,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error {
 	loadGitFrom(cfg)
 	loadMirrorFrom(cfg)
 	loadMarkupFrom(cfg)
+	loadGlobalLockFrom(cfg)
 	loadOtherFrom(cfg)
 	return nil
 }