From d15388cb25f94e889dd61f6b8613d06ca1a9d3af Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Aug 2023 14:02:38 +0800 Subject: [PATCH 1/5] Use an abstract lock layer to allow distributed lock between multiple Gitea instances --- assets/go-licenses.json | 5 + custom/conf/app.example.ini | 6 + .../config-cheat-sheet.en-us.md | 5 + .../config-cheat-sheet.zh-cn.md | 7 + go.mod | 1 + go.sum | 24 ++++ modules/setting/setting.go | 1 + modules/setting/sync.go | 37 +++++ modules/setting/sync_test.go | 35 +++++ modules/sync/exclusive_pool.go | 69 ---------- modules/sync/lock.go | 128 ++++++++++++++++++ modules/sync/lock_test.go | 26 ++++ modules/sync/status_pool.go | 57 -------- modules/sync/status_pool_test.go | 31 ----- services/cron/cron.go | 4 - services/cron/tasks.go | 14 +- services/pull/check.go | 18 ++- services/pull/merge.go | 25 +++- services/pull/pull.go | 16 ++- services/pull/update.go | 13 +- services/repository/transfer.go | 34 +++-- services/wiki/wiki.go | 29 +++- 22 files changed, 392 insertions(+), 193 deletions(-) create mode 100644 modules/setting/sync.go create mode 100644 modules/setting/sync_test.go delete mode 100644 modules/sync/exclusive_pool.go create mode 100644 modules/sync/lock.go create mode 100644 modules/sync/lock_test.go delete mode 100644 modules/sync/status_pool.go delete mode 100644 modules/sync/status_pool_test.go diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 3a0da33a0a382..8caa42e750ac0 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -454,6 +454,11 @@ "path": "github.com/go-ldap/ldap/v3/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com)\nPortions copyright (c) 2015-2016 go-ldap Authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, + { + "name": "github.com/go-redsync/redsync/v4", + "path": "github.com/go-redsync/redsync/v4/LICENSE", + "licenseText": "Copyright (c) 2021, Mahmud Ridwan\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the Redsync nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "github.com/go-sql-driver/mysql", "path": "github.com/go-sql-driver/mysql/LICENSE", diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index cfaf91cddb7f2..b3ff35f469cc9 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2573,3 +2573,9 @@ LEVEL = Info ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; storage type ;STORAGE_TYPE = local + +;[sync] +;; Lock service type, could be memory or redis +;LOCK_SERVICE_TYPE = memory +;; Ignored by memory type, for redis, it likes "addrs=127.0.0.1:6379 db=0" +;LOCK_SERVICE_CONN_STR = diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index 71ae4f2e30bd2..e3148739febc1 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -1395,6 +1395,11 @@ However, later updates removed those options, and now the only options are `gith However, if you want to use actions from other git server, you can use a complete URL in `uses` field, it's supported by Gitea (but not GitHub). Like `uses: https://gitea.com/actions/checkout@v3` or `uses: http://your-git-server/actions/checkout@v3`. +## Sync (`sync`) + +- `LOCK_SERVICE_TYPE`: **memory**: Lock service type, could be `memory` or `redis` +- `LOCK_SERVICE_CONN_STR`: **\**: Ignored when `LOCK_SERVICE_TYPE` is `memory` type, for `redis`, it likes `redis://127.0.0.1:6379/0` + ## Other (`other`) - `SHOW_FOOTER_VERSION`: **true**: Show Gitea and Go version information in the footer. diff --git a/docs/content/administration/config-cheat-sheet.zh-cn.md b/docs/content/administration/config-cheat-sheet.zh-cn.md index 87e45aa6934d0..21c2bb5b9155d 100644 --- a/docs/content/administration/config-cheat-sheet.zh-cn.md +++ b/docs/content/administration/config-cheat-sheet.zh-cn.md @@ -1349,6 +1349,13 @@ PROXY_HOSTS = *.github.com 但是，如果您想要使用其他 Git 服务器中的操作，您可以在 `uses` 字段中使用完整的 URL，Gitea 支持此功能（GitHub 不支持）。 例如 `uses: https://gitea.com/actions/checkout@v3` 或 `uses: http://your-git-server/actions/checkout@v3`。 +## Sync (`sync`) + +- `LOCK_SERVICE_TYPE`: **memory**: 锁服务类型，可以是 `memory` 或 `redis` +- `LOCK_SERVICE_CONN_STR`: **\**: 如果 `LOCK_SERVICE_TYPE` 是 `memory`，则忽略，如果是 `redis`，则支持形式如 `redis://127.0.0.1:6379/0` + +## Other (`other`) + ## 其他 (`other`) - `SHOW_FOOTER_VERSION`: **true**: 在页面底部显示Gitea的版本。 diff --git a/go.mod b/go.mod index b1e31c3faaff9..16e207dd14835 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git/v5 v5.7.0 github.com/go-ldap/ldap/v3 v3.4.5 + github.com/go-redsync/redsync/v4 v4.8.1 github.com/go-sql-driver/mysql v1.7.1 github.com/go-swagger/go-swagger v0.30.5 github.com/go-testfixtures/testfixtures/v3 v3.9.0 diff --git a/go.sum b/go.sum index 5e942457a5571..1ccd538ec30a2 100644 --- a/go.sum +++ b/go.sum @@ -198,7 +198,9 @@ github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bradfitz/gomemcache v0.0.0-20230611145640-acc696258285 h1:Dr+ezPI5ivhMn/3WOoB86XzMhie146DNaBbhaQWZHMY= github.com/bradfitz/gomemcache v0.0.0-20230611145640-acc696258285/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bufbuild/connect-go v1.9.0 h1:JIgAeNuFpo+SUPfU19Yt5TcWlznsN5Bv10/gI/6Pjoc= github.com/bufbuild/connect-go v1.9.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= @@ -211,6 +213,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chi-middleware/proxy v1.1.1 h1:4HaXUp8o2+bhHr1OhVy+VjN0+L7/07JDcn6v7YrTjrQ= @@ -401,7 +404,15 @@ github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= +github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-redis/redis/v8 v8.4.0/go.mod h1:A1tbYoHSa1fXwN+//ljcCYYJeLmVrwL9hbQN45Jdy0M= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM= +github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= @@ -494,6 +505,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -501,6 +513,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -514,6 +528,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= @@ -828,15 +843,19 @@ github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCz github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -880,6 +899,7 @@ github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH github.com/quasoft/websspi v1.1.2 h1:/mA4w0LxWlE3novvsoEL6BBA1WnjJATbjkh1kFrTidw= github.com/quasoft/websspi v1.1.2/go.mod h1:HmVdl939dQ0WIXZhyik+ARdI03M6bQzaSEKcgpFmewk= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= @@ -983,6 +1003,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= +github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -1182,6 +1204,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1205,6 +1228,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d444d9a0175c6..511397a4de0db 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -152,6 +152,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error { loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadOtherFrom(cfg) + loadSyncFrom(cfg) return nil } diff --git a/modules/setting/sync.go b/modules/setting/sync.go new file mode 100644 index 0000000000000..b81ab87abfb89 --- /dev/null +++ b/modules/setting/sync.go @@ -0,0 +1,37 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/nosql" +) + +// Sync represents configuration of sync +var Sync = struct { + LockServiceType string + LockServiceConnStr string +}{ + LockServiceType: "memory", +} + +func loadSyncFrom(rootCfg ConfigProvider) { + sec := rootCfg.Section("sync") + Sync.LockServiceType = sec.Key("LOCK_SERVICE_TYPE").MustString("memory") + switch Sync.LockServiceType { + case "memory": + case "redis": + connStr := sec.Key("LOCK_SERVICE_CONN_STR").String() + if connStr == "" { + log.Fatal("LOCK_SERVICE_CONN_STR is empty for redis") + } + u := nosql.ToRedisURI(connStr) + if u == nil { + log.Fatal("LOCK_SERVICE_CONN_STR %s is not right for redis", connStr) + } + Sync.LockServiceConnStr = connStr + default: + log.Fatal("Unknown sync lock service type: %s", Sync.LockServiceType) + } +} diff --git a/modules/setting/sync_test.go b/modules/setting/sync_test.go new file mode 100644 index 0000000000000..a90e6196798c9 --- /dev/null +++ b/modules/setting/sync_test.go @@ -0,0 +1,35 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadSyncConfig(t *testing.T) { + t.Run("DefaultSyncConfig", func(t *testing.T) { + iniStr := `` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + loadSyncFrom(cfg) + assert.EqualValues(t, "memory", Sync.LockServiceType) + }) + + t.Run("RedisSyncConfig", func(t *testing.T) { + iniStr := ` +[sync] +LOCK_SERVICE_TYPE = redis +LOCK_SERVICE_CONN_STR = addrs=127.0.0.1:6379 db=0 +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + loadSyncFrom(cfg) + assert.EqualValues(t, "redis", Sync.LockServiceType) + assert.EqualValues(t, "addrs=127.0.0.1:6379 db=0", Sync.LockServiceConnStr) + }) +} diff --git a/modules/sync/exclusive_pool.go b/modules/sync/exclusive_pool.go deleted file mode 100644 index fbfc1f22924c6..0000000000000 --- a/modules/sync/exclusive_pool.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2016 The Gogs Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package sync - -import ( - "sync" -) - -// ExclusivePool is a pool of non-identical instances -// that only one instance with same identity is in the pool at a time. -// In other words, only instances with different identities can be in -// the pool the same time. If another instance with same identity tries -// to get into the pool, it hangs until previous instance left the pool. -// -// This pool is particularly useful for performing tasks on same resource -// on the file system in different goroutines. -type ExclusivePool struct { - lock sync.Mutex - - // pool maintains locks for each instance in the pool. - pool map[string]*sync.Mutex - - // count maintains the number of times an instance with same identity checks in - // to the pool, and should be reduced to 0 (removed from map) by checking out - // with same number of times. - // The purpose of count is to delete lock when count down to 0 and recycle memory - // from map object. - count map[string]int -} - -// NewExclusivePool initializes and returns a new ExclusivePool object. -func NewExclusivePool() *ExclusivePool { - return &ExclusivePool{ - pool: make(map[string]*sync.Mutex), - count: make(map[string]int), - } -} - -// CheckIn checks in an instance to the pool and hangs while instance -// with same identity is using the lock. -func (p *ExclusivePool) CheckIn(identity string) { - p.lock.Lock() - - lock, has := p.pool[identity] - if !has { - lock = &sync.Mutex{} - p.pool[identity] = lock - } - p.count[identity]++ - - p.lock.Unlock() - lock.Lock() -} - -// CheckOut checks out an instance from the pool and releases the lock -// to let other instances with same identity to grab the lock. -func (p *ExclusivePool) CheckOut(identity string) { - p.lock.Lock() - defer p.lock.Unlock() - - p.pool[identity].Unlock() - if p.count[identity] == 1 { - delete(p.pool, identity) - delete(p.count, identity) - } else { - p.count[identity]-- - } -} diff --git a/modules/sync/lock.go b/modules/sync/lock.go new file mode 100644 index 0000000000000..5720ed17fc748 --- /dev/null +++ b/modules/sync/lock.go @@ -0,0 +1,128 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package sync + +import ( + "context" + "sync" + "time" + + "code.gitea.io/gitea/modules/nosql" + "code.gitea.io/gitea/modules/setting" + + redsync "github.com/go-redsync/redsync/v4" + goredis "github.com/go-redsync/redsync/v4/redis/goredis/v9" +) + +type Locker interface { + Lock() error + TryLock() (bool, error) + Unlock() (bool, error) +} + +type LockService interface { + GetLock(name string) Locker +} + +type memoryLock struct { + mutex sync.Mutex +} + +func (r *memoryLock) Lock() error { + r.mutex.Lock() + return nil +} + +func (r *memoryLock) TryLock() (bool, error) { + return r.mutex.TryLock(), nil +} + +func (r *memoryLock) Unlock() (bool, error) { + r.mutex.Unlock() + return true, nil +} + +var _ Locker = &memoryLock{} + +type memoryLockService struct { + syncMap sync.Map +} + +var _ LockService = &memoryLockService{} + +func newMemoryLockService() *memoryLockService { + return &memoryLockService{ + syncMap: sync.Map{}, + } +} + +func (l *memoryLockService) GetLock(name string) Locker { + v, _ := l.syncMap.LoadOrStore(name, &memoryLock{}) + return v.(*memoryLock) +} + +type redisLockService struct { + rs *redsync.Redsync +} + +var _ LockService = &redisLockService{} + +func newRedisLockService(connection string) *redisLockService { + client := nosql.GetManager().GetRedisClient(connection) + + pool := goredis.NewPool(client) // or, pool := redigo.NewPool(...) + + // Create an instance of redisync to be used to obtain a mutual exclusion + // lock. + rs := redsync.New(pool) + + return &redisLockService{ + rs: rs, + } +} + +type redisLock struct { + mutex *redsync.Mutex +} + +func (r *redisLockService) GetLock(name string) Locker { + return &redisLock{mutex: r.rs.NewMutex(name)} +} + +func (r *redisLock) Lock() error { + return r.mutex.Lock() +} + +func (r *redisLock) TryLock() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := r.mutex.LockContext(ctx); err != nil { + return false, err + } + return true, nil +} + +func (r *redisLock) Unlock() (bool, error) { + return r.mutex.Unlock() +} + +var ( + syncOnce sync.Once + lockService LockService +) + +func getLockService() LockService { + syncOnce.Do(func() { + if setting.Sync.LockServiceType == "redis" { + lockService = newRedisLockService(setting.Sync.LockServiceConnStr) + } else { + lockService = newMemoryLockService() + } + }) + return lockService +} + +func GetLock(name string) Locker { + return getLockService().GetLock(name) +} diff --git a/modules/sync/lock_test.go b/modules/sync/lock_test.go new file mode 100644 index 0000000000000..6adc287eb14d8 --- /dev/null +++ b/modules/sync/lock_test.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package sync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Lock(t *testing.T) { + locker := GetLock("test") + assert.NoError(t, locker.Lock()) + locker.Unlock() + + locked1, err1 := locker.TryLock() + assert.NoError(t, err1) + assert.True(t, locked1) + + locked2, err2 := locker.TryLock() + assert.NoError(t, err2) + assert.False(t, locked2) + + locker.Unlock() +} diff --git a/modules/sync/status_pool.go b/modules/sync/status_pool.go deleted file mode 100644 index 6f075d54b79db..0000000000000 --- a/modules/sync/status_pool.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2016 The Gogs Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package sync - -import ( - "sync" - - "code.gitea.io/gitea/modules/container" -) - -// StatusTable is a table maintains true/false values. -// -// This table is particularly useful for un/marking and checking values -// in different goroutines. -type StatusTable struct { - lock sync.RWMutex - pool container.Set[string] -} - -// NewStatusTable initializes and returns a new StatusTable object. -func NewStatusTable() *StatusTable { - return &StatusTable{ - pool: make(container.Set[string]), - } -} - -// StartIfNotRunning sets value of given name to true if not already in pool. -// Returns whether set value was set to true -func (p *StatusTable) StartIfNotRunning(name string) bool { - p.lock.Lock() - added := p.pool.Add(name) - p.lock.Unlock() - return added -} - -// Start sets value of given name to true in the pool. -func (p *StatusTable) Start(name string) { - p.lock.Lock() - p.pool.Add(name) - p.lock.Unlock() -} - -// Stop sets value of given name to false in the pool. -func (p *StatusTable) Stop(name string) { - p.lock.Lock() - p.pool.Remove(name) - p.lock.Unlock() -} - -// IsRunning checks if value of given name is set to true in the pool. -func (p *StatusTable) IsRunning(name string) bool { - p.lock.RLock() - exists := p.pool.Contains(name) - p.lock.RUnlock() - return exists -} diff --git a/modules/sync/status_pool_test.go b/modules/sync/status_pool_test.go deleted file mode 100644 index e2e48862f581f..0000000000000 --- a/modules/sync/status_pool_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package sync - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_StatusTable(t *testing.T) { - table := NewStatusTable() - - assert.False(t, table.IsRunning("xyz")) - - table.Start("xyz") - assert.True(t, table.IsRunning("xyz")) - - assert.False(t, table.StartIfNotRunning("xyz")) - assert.True(t, table.IsRunning("xyz")) - - table.Stop("xyz") - assert.False(t, table.IsRunning("xyz")) - - assert.True(t, table.StartIfNotRunning("xyz")) - assert.True(t, table.IsRunning("xyz")) - - table.Stop("xyz") - assert.False(t, table.IsRunning("xyz")) -} diff --git a/services/cron/cron.go b/services/cron/cron.go index e3f31d08f0c90..fdcba35494f4d 100644 --- a/services/cron/cron.go +++ b/services/cron/cron.go @@ -11,7 +11,6 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/translation" "github.com/go-co-op/gocron" @@ -19,9 +18,6 @@ import ( var scheduler = gocron.NewScheduler(time.Local) -// Prevent duplicate running tasks. -var taskStatusTable = sync.NewStatusTable() - // NewContext begins cron tasks // Each cron task is run within the shutdown context as a running server // AtShutdown the cron server is stopped diff --git a/services/cron/tasks.go b/services/cron/tasks.go index ea1925c26c738..b7bf99cdabc40 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + sync_module "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/translation" ) @@ -70,17 +71,24 @@ func (t *Task) Run() { // RunWithUser will run the task incrementing the cron counter at the time with User func (t *Task) RunWithUser(doer *user_model.User, config Config) { - if !taskStatusTable.StartIfNotRunning(t.Name) { + // the same task can run only once at the same time + lock := sync_module.GetLock(fmt.Sprintf("cron_tasks_%s", t.Name)) + if locked, err := lock.TryLock(); err != nil { + log.Error("Unable to lock cron task %s Error: %v", t.Name, err) + return + } else if locked { return } - t.lock.Lock() + defer t.lock.Lock() if config == nil { config = t.config } t.ExecTimes++ t.lock.Unlock() defer func() { - taskStatusTable.Stop(t.Name) + if _, err := lock.Unlock(); err != nil { + log.Error("Unable to unlock cron task %s Error: %v", t.Name, err) + } if err := recover(); err != nil { // Recover a panic within the combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) diff --git a/services/pull/check.go b/services/pull/check.go index ec898201bb66a..834dbe9344466 100644 --- a/services/pull/check.go +++ b/services/pull/check.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" + sync_module "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" asymkey_service "code.gitea.io/gitea/services/asymkey" ) @@ -330,9 +331,22 @@ func handler(items ...string) []string { return nil } +func getPullWorkingLockKey(prID int64) string { + return fmt.Sprintf("pull_working_%d", prID) +} + func testPR(id int64) { - pullWorkingPool.CheckIn(fmt.Sprint(id)) - defer pullWorkingPool.CheckOut(fmt.Sprint(id)) + lock := sync_module.GetLock(getPullWorkingLockKey(id)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("Test PR[%d] from patch checking queue", id)) defer finished() diff --git a/services/pull/merge.go b/services/pull/merge.go index 7051fd9eda24d..4d737c00113e4 100644 --- a/services/pull/merge.go +++ b/services/pull/merge.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/references" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" + sync_module "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" issue_service "code.gitea.io/gitea/services/issue" ) @@ -156,8 +157,16 @@ func Merge(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.U return fmt.Errorf("unable to load head repo: %w", err) } - pullWorkingPool.CheckIn(fmt.Sprint(pr.ID)) - defer pullWorkingPool.CheckOut(fmt.Sprint(pr.ID)) + lock := sync_module.GetLock(getPullWorkingLockKey(pr.ID)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return fmt.Errorf("lock.Lock: %w", err) + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() // Removing an auto merge pull and ignore if not exist // FIXME: is this the correct point to do this? Shouldn't this be after IsMergeStyleAllowed? @@ -465,8 +474,16 @@ func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullReques // MergedManually mark pr as merged manually func MergedManually(pr *issues_model.PullRequest, doer *user_model.User, baseGitRepo *git.Repository, commitID string) error { - pullWorkingPool.CheckIn(fmt.Sprint(pr.ID)) - defer pullWorkingPool.CheckOut(fmt.Sprint(pr.ID)) + lock := sync_module.GetLock(getPullWorkingLockKey(pr.ID)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return fmt.Errorf("lock.Lock: %w", err) + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { if err := pr.LoadBaseRepo(ctx); err != nil { diff --git a/services/pull/pull.go b/services/pull/pull.go index 0b6194b1436d8..e12ffaae46c0e 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -33,9 +33,6 @@ import ( issue_service "code.gitea.io/gitea/services/issue" ) -// TODO: use clustered lock (unique queue? or *abuse* cache) -var pullWorkingPool = sync.NewExclusivePool() - // NewPullRequest creates new pull request with labels for repository. func NewPullRequest(ctx context.Context, repo *repo_model.Repository, issue *issues_model.Issue, labelIDs []int64, uuids []string, pr *issues_model.PullRequest, assigneeIDs []int64) error { prCtx, cancel, err := createTemporaryRepoForPR(ctx, pr) @@ -167,8 +164,17 @@ func NewPullRequest(ctx context.Context, repo *repo_model.Repository, issue *iss // ChangeTargetBranch changes the target branch of this pull request, as the given user. func ChangeTargetBranch(ctx context.Context, pr *issues_model.PullRequest, doer *user_model.User, targetBranch string) (err error) { - pullWorkingPool.CheckIn(fmt.Sprint(pr.ID)) - defer pullWorkingPool.CheckOut(fmt.Sprint(pr.ID)) + lock := sync.GetLock(getPullWorkingLockKey(pr.ID)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return fmt.Errorf("lock.Lock: %w", err) + } + defer func() { + _, err := lock.Unlock() + if err != nil { + log.Error("lock.Unlock(): %v", err) + } + }() // Current target branch is already the same if pr.BaseBranch == targetBranch { diff --git a/services/pull/update.go b/services/pull/update.go index bc8c4a25e5f61..7a9a8c914f88b 100644 --- a/services/pull/update.go +++ b/services/pull/update.go @@ -15,6 +15,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + sync_module "code.gitea.io/gitea/modules/sync" ) // Update updates pull request with base branch. @@ -24,8 +25,16 @@ func Update(ctx context.Context, pr *issues_model.PullRequest, doer *user_model. return fmt.Errorf("update of agit flow pull request's head branch is unsupported") } - pullWorkingPool.CheckIn(fmt.Sprint(pr.ID)) - defer pullWorkingPool.CheckOut(fmt.Sprint(pr.ID)) + lock := sync_module.GetLock(getPullWorkingLockKey(pr.ID)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return fmt.Errorf("lock.Lock: %w", err) + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() diffCount, err := GetDiverging(ctx, pr) if err != nil { diff --git a/services/repository/transfer.go b/services/repository/transfer.go index b9b26f314c2c3..b2fa5ea528854 100644 --- a/services/repository/transfer.go +++ b/services/repository/transfer.go @@ -19,9 +19,9 @@ import ( "code.gitea.io/gitea/modules/sync" ) -// repoWorkingPool represents a working pool to order the parallel changes to the same repository -// TODO: use clustered lock (unique queue? or *abuse* cache) -var repoWorkingPool = sync.NewExclusivePool() +func getRepoWorkingLockKey(repoID int64) string { + return fmt.Sprintf("repo_working_%d", repoID) +} // TransferOwnership transfers all corresponding setting from old user to new one. func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { @@ -36,12 +36,20 @@ func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, rep oldOwner := repo.Owner - repoWorkingPool.CheckIn(fmt.Sprint(repo.ID)) + lock := sync.GetLock(getRepoWorkingLockKey(repo.ID)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return fmt.Errorf("lock.Lock: %w", err) + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() + if err := models.TransferOwnership(doer, newOwner.Name, repo); err != nil { - repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) return err } - repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) newRepo, err := repo_model.GetRepositoryByID(ctx, repo.ID) if err != nil { @@ -69,12 +77,20 @@ func ChangeRepositoryName(ctx context.Context, doer *user_model.User, repo *repo // repo so that we can atomically rename the repo path and updates the // local copy's origin accordingly. - repoWorkingPool.CheckIn(fmt.Sprint(repo.ID)) + lock := sync.GetLock(getRepoWorkingLockKey(repo.ID)) + if err := lock.Lock(); err != nil { + log.Error("lock.Lock(): %v", err) + return fmt.Errorf("lock.Lock: %w", err) + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() + if err := repo_model.ChangeRepositoryName(doer, repo, newRepoName); err != nil { - repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) return err } - repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) repo.Name = newRepoName notification.NotifyRenameRepository(ctx, doer, repo, oldRepoName) diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index c0183dd2b590a..3018ea32ea958 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -21,9 +21,6 @@ import ( asymkey_service "code.gitea.io/gitea/services/asymkey" ) -// TODO: use clustered lock (unique queue? or *abuse* cache) -var wikiWorkingPool = sync.NewExclusivePool() - const ( DefaultRemote = "origin" DefaultBranch = "master" @@ -77,13 +74,24 @@ func prepareGitPath(gitRepo *git.Repository, wikiPath WebPath) (bool, string, er return foundEscaped, gitPath, nil } +func getWikiWorkingLockKey(repoID int64) string { + return fmt.Sprintf("wiki_working_%d", repoID) +} + // updateWikiPage adds a new page or edits an existing page in repository wiki. func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, oldWikiName, newWikiName WebPath, content, message string, isNew bool) (err error) { if err = validateWebPath(newWikiName); err != nil { return err } - wikiWorkingPool.CheckIn(fmt.Sprint(repo.ID)) - defer wikiWorkingPool.CheckOut(fmt.Sprint(repo.ID)) + lock := sync.GetLock(getWikiWorkingLockKey(repo.ID)) + if err := lock.Lock(); err != nil { + return err + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() if err = InitWiki(ctx, repo); err != nil { return fmt.Errorf("InitWiki: %w", err) @@ -238,8 +246,15 @@ func EditWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model.R // DeleteWikiPage deletes a wiki page identified by its path. func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, wikiName WebPath) (err error) { - wikiWorkingPool.CheckIn(fmt.Sprint(repo.ID)) - defer wikiWorkingPool.CheckOut(fmt.Sprint(repo.ID)) + lock := sync.GetLock(getWikiWorkingLockKey(repo.ID)) + if err := lock.Lock(); err != nil { + return err + } + defer func() { + if _, err := lock.Unlock(); err != nil { + log.Error("lock.Unlock: %v", err) + } + }() if err = InitWiki(ctx, repo); err != nil { return fmt.Errorf("InitWiki: %w", err) From 45e783ee4ec98825c55d12d115147aae9fd156c6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Aug 2023 18:55:32 +0800 Subject: [PATCH 2/5] Fix bug --- services/cron/tasks.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index b7bf99cdabc40..a05f6d131f71b 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -73,13 +73,15 @@ func (t *Task) Run() { func (t *Task) RunWithUser(doer *user_model.User, config Config) { // the same task can run only once at the same time lock := sync_module.GetLock(fmt.Sprintf("cron_tasks_%s", t.Name)) - if locked, err := lock.TryLock(); err != nil { + if success, err := lock.TryLock(); err != nil { log.Error("Unable to lock cron task %s Error: %v", t.Name, err) return - } else if locked { + } else if !success { + // get lock failed, so that there must be another task are running. return } - defer t.lock.Lock() + + t.lock.Lock() if config == nil { config = t.config } From 7d568f722423901435263dc35eab6726c69870e7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 11:48:03 +0800 Subject: [PATCH 3/5] Update modules/setting/sync.go Co-authored-by: delvh --- modules/setting/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/sync.go b/modules/setting/sync.go index b81ab87abfb89..20d339253f37e 100644 --- a/modules/setting/sync.go +++ b/modules/setting/sync.go @@ -28,7 +28,7 @@ func loadSyncFrom(rootCfg ConfigProvider) { } u := nosql.ToRedisURI(connStr) if u == nil { - log.Fatal("LOCK_SERVICE_CONN_STR %s is not right for redis", connStr) + log.Fatal("LOCK_SERVICE_CONN_STR %s is not a valid redis connection string", connStr) } Sync.LockServiceConnStr = connStr default: From 974f6bd958726774ff0d0cdfb0b9cafd56bb7c16 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 11:48:13 +0800 Subject: [PATCH 4/5] Update custom/conf/app.example.ini Co-authored-by: delvh --- custom/conf/app.example.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b3ff35f469cc9..6f7799beff161 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2577,5 +2577,5 @@ LEVEL = Info ;[sync] ;; Lock service type, could be memory or redis ;LOCK_SERVICE_TYPE = memory -;; Ignored by memory type, for redis, it likes "addrs=127.0.0.1:6379 db=0" +;; Ignored for the "memory" type. For "redis" use something like `redis://127.0.0.1:6379/0` ;LOCK_SERVICE_CONN_STR = From 8a62284c4a59409ad104a88cd460f3b6d3f03834 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Aug 2023 11:48:32 +0800 Subject: [PATCH 5/5] Update docs/content/administration/config-cheat-sheet.en-us.md Co-authored-by: delvh --- docs/content/administration/config-cheat-sheet.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index e3148739febc1..621aec8861ad2 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -1398,7 +1398,7 @@ Like `uses: https://gitea.com/actions/checkout@v3` or `uses: http://your-git-ser ## Sync (`sync`) - `LOCK_SERVICE_TYPE`: **memory**: Lock service type, could be `memory` or `redis` -- `LOCK_SERVICE_CONN_STR`: **\**: Ignored when `LOCK_SERVICE_TYPE` is `memory` type, for `redis`, it likes `redis://127.0.0.1:6379/0` +- `LOCK_SERVICE_CONN_STR`: **\**: Ignored when `LOCK_SERVICE_TYPE` is `memory`. For `redis` use something like `redis://127.0.0.1:6379/0` ## Other (`other`)