Merge remote-tracking branch 'giteaofficial/main'

* giteaofficial/main: Store webhook event in database (go-gitea#29145) Fix bug hidden on CI and make ci failed if tests failure (go-gitea#29254)
zjjhot · Mar 8, 2024 · d8bf5b0 · d8bf5b0
2 parents d057e11 + 26653b1
commit d8bf5b0
Show file tree

Hide file tree

Showing 48 changed files with 1,887 additions and 1,591 deletions.
diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml
@@ -49,7 +49,10 @@ jobs:
       - run: make backend
         env:
           TAGS: bindata
-      - run: make test-pgsql-migration test-pgsql
+      - name: run migration tests
+        run: make test-pgsql-migration
+      - name: run tests
+        run: make test-pgsql
         timeout-minutes: 50
         env:
           TAGS: bindata gogit
@@ -72,7 +75,10 @@ jobs:
       - run: make backend
         env:
           TAGS: bindata gogit sqlite sqlite_unlock_notify
-      - run: make test-sqlite-migration test-sqlite
+      - name: run migration tests
+        run: make test-sqlite-migration
+      - name: run tests
+        run: make test-sqlite
         timeout-minutes: 50
         env:
           TAGS: bindata gogit sqlite sqlite_unlock_notify
@@ -175,8 +181,10 @@ jobs:
       - run: make backend
         env:
           TAGS: bindata
+      - name: run migration tests
+        run: make test-mysql-migration
       - name: run tests
-        run: make test-mysql-migration integration-test-coverage
+        run: make integration-test-coverage
         env:
           TAGS: bindata
           RACE_ENABLED: true
@@ -208,7 +216,9 @@ jobs:
       - run: make backend
         env:
           TAGS: bindata
-      - run: make test-mssql-migration test-mssql
+      - run: make test-mssql-migration
+      - name: run tests
+        run: make test-mssql
         timeout-minutes: 50
         env:
           TAGS: bindata

diff --git a/Makefile b/Makefile
@@ -115,6 +115,7 @@ LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
 
 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
 GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
+MIGRATE_TEST_PACKAGES ?= $(shell $(GO) list code.gitea.io/gitea/models/migrations/...)
 
 FOMANTIC_WORK_DIR := web_src/fomantic
 
@@ -710,40 +711,31 @@ migrations.sqlite.test: $(GO_SOURCES) generate-ini-sqlite
 
 .PHONY: migrations.individual.mysql.test
 migrations.individual.mysql.test: $(GO_SOURCES)
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
-		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mysql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
-	done
+	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mysql.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES)
 
 .PHONY: migrations.individual.sqlite.test\#%
 migrations.individual.sqlite.test\#%: $(GO_SOURCES) generate-ini-sqlite
 	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$*
 
 .PHONY: migrations.individual.pgsql.test
 migrations.individual.pgsql.test: $(GO_SOURCES)
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
-		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
-	done
+	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES)
 
 .PHONY: migrations.individual.pgsql.test\#%
 migrations.individual.pgsql.test\#%: $(GO_SOURCES) generate-ini-pgsql
 	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$*
 
-
 .PHONY: migrations.individual.mssql.test
 migrations.individual.mssql.test: $(GO_SOURCES) generate-ini-mssql
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
-		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg -test.failfast; \
-	done
+	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES)
 
 .PHONY: migrations.individual.mssql.test\#%
 migrations.individual.mssql.test\#%: $(GO_SOURCES) generate-ini-mssql
 	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' code.gitea.io/gitea/models/migrations/$*
 
 .PHONY: migrations.individual.sqlite.test
 migrations.individual.sqlite.test: $(GO_SOURCES) generate-ini-sqlite
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
-		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
-	done
+	GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' -p 1 $(MIGRATE_TEST_PACKAGES)
 
 .PHONY: migrations.individual.sqlite.test\#%
 migrations.individual.sqlite.test\#%: $(GO_SOURCES) generate-ini-sqlite

diff --git a/models/fixtures/hook_task.yml b/models/fixtures/hook_task.yml
@@ -3,3 +3,35 @@
   hook_id: 1
   uuid: uuid1
   is_delivered: true
+  is_succeed: false
+  request_content: >
+    {
+      "url": "/matrix-delivered",
+      "http_method":"PUT",
+      "headers": {
+        "X-Head": "42"
+      },
+      "body": "{}"
+    }
+
+-
+  id: 2
+  hook_id: 1
+  uuid: uuid2
+  is_delivered: false
+
+-
+  id: 3
+  hook_id: 1
+  uuid: uuid3
+  is_delivered: true
+  is_succeed: true
+  payload_content: '{"key":"value"}' # legacy task, payload saved in payload_content (and not in request_content)
+  request_content: >
+    {
+      "url": "/matrix-success",
+      "http_method":"PUT",
+      "headers": {
+        "X-Head": "42"
+      }
+    }
diff --git a/models/migrations/base/db_test.go b/models/migrations/base/db_test.go
@@ -36,12 +36,14 @@ func Test_DropTableColumns(t *testing.T) {
 		"updated_unix",
 	}
 
+	x.SetMapper(names.GonicMapper{})
+
 	for i := range columns {
-		x.SetMapper(names.GonicMapper{})
 		if err := x.Sync(new(DropTest)); err != nil {
 			t.Errorf("unable to create DropTest table: %v", err)
 			return
 		}
+
 		sess := x.NewSession()
 		if err := sess.Begin(); err != nil {
 			sess.Close()
@@ -64,7 +66,6 @@ func Test_DropTableColumns(t *testing.T) {
 			return
 		}
 		for j := range columns[i+1:] {
-			x.SetMapper(names.GonicMapper{})
 			if err := x.Sync(new(DropTest)); err != nil {
 				t.Errorf("unable to create DropTest table: %v", err)
 				return

diff --git a/models/migrations/fixtures/Test_AddIssueResourceIndexTable/issue.yml b/models/migrations/fixtures/Test_AddIssueResourceIndexTable/issue.yml
@@ -0,0 +1,4 @@
+-
+  id: 1
+  repo_id: 1
+  index: 1
diff --git a/models/migrations/fixtures/Test_AddRepoIDForAttachment/attachment.yml b/models/migrations/fixtures/Test_AddRepoIDForAttachment/attachment.yml
@@ -0,0 +1,11 @@
+-
+  id: 1
+  uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11
+  issue_id: 1
+  release_id: 0
+
+-
+  id: 2
+  uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12
+  issue_id: 0
+  release_id: 1
diff --git a/models/migrations/fixtures/Test_AddRepoIDForAttachment/issue.yml b/models/migrations/fixtures/Test_AddRepoIDForAttachment/issue.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  repo_id: 1
diff --git a/models/migrations/fixtures/Test_AddRepoIDForAttachment/release.yml b/models/migrations/fixtures/Test_AddRepoIDForAttachment/release.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  repo_id: 1
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/comment.yml b/models/migrations/fixtures/Test_RepositoryFormat/comment.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  commit_sha: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/commit_status.yml b/models/migrations/fixtures/Test_RepositoryFormat/commit_status.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  context_hash: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/pull_request.yml b/models/migrations/fixtures/Test_RepositoryFormat/pull_request.yml
@@ -0,0 +1,5 @@
+-
+  id: 1
+  commit_sha: 19fe5caf872476db265596eaac1dc35ad1c6422d
+  merge_base: 19fe5caf872476db265596eaac1dc35ad1c6422d
+  merged_commit_id: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/release.yml b/models/migrations/fixtures/Test_RepositoryFormat/release.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  sha1: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/repo_archiver.yml b/models/migrations/fixtures/Test_RepositoryFormat/repo_archiver.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  commit_id: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/repo_indexer_status.yml b/models/migrations/fixtures/Test_RepositoryFormat/repo_indexer_status.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  commit_sha: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_RepositoryFormat/review_state.yml b/models/migrations/fixtures/Test_RepositoryFormat/review_state.yml
@@ -0,0 +1,3 @@
+-
+  id: 1
+  commit_sha: 19fe5caf872476db265596eaac1dc35ad1c6422d
diff --git a/models/migrations/fixtures/Test_UpdateBadgeColName/badge.yml b/models/migrations/fixtures/Test_UpdateBadgeColName/badge.yml
@@ -0,0 +1,4 @@
+-
+  id: 1
+  description: the badge
+  image_url: https://gitea.com/myimage.png
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
@@ -564,6 +564,8 @@ var migrations = []Migration{
 	NewMigration("Add user_blocking table", v1_22.AddUserBlockingTable),
 	// v289 -> v290
 	NewMigration("Add default_wiki_branch to repository table", v1_22.AddDefaultWikiBranch),
+	// v290 -> v291
+	NewMigration("Add PayloadVersion to HookTask", v1_22.AddPayloadVersionToHookTaskTable),
 }
 
 // GetCurrentDBVersion returns the current db version

diff --git a/models/migrations/v1_16/v193_test.go b/models/migrations/v1_16/v193_test.go
@@ -15,7 +15,6 @@ func Test_AddRepoIDForAttachment(t *testing.T) {
 	type Attachment struct {
 		ID         int64  `xorm:"pk autoincr"`
 		UUID       string `xorm:"uuid UNIQUE"`
-		RepoID     int64  `xorm:"INDEX"` // this should not be zero
 		IssueID    int64  `xorm:"INDEX"` // maybe zero when creating
 		ReleaseID  int64  `xorm:"INDEX"` // maybe zero when creating
 		UploaderID int64  `xorm:"INDEX DEFAULT 0"`
@@ -44,25 +43,34 @@ func Test_AddRepoIDForAttachment(t *testing.T) {
 		return
 	}
 
-	var issueAttachments []*Attachment
-	err := x.Where("issue_id > 0").Find(&issueAttachments)
+	type NewAttachment struct {
+		ID         int64  `xorm:"pk autoincr"`
+		UUID       string `xorm:"uuid UNIQUE"`
+		RepoID     int64  `xorm:"INDEX"` // this should not be zero
+		IssueID    int64  `xorm:"INDEX"` // maybe zero when creating
+		ReleaseID  int64  `xorm:"INDEX"` // maybe zero when creating
+		UploaderID int64  `xorm:"INDEX DEFAULT 0"`
+	}
+
+	var issueAttachments []*NewAttachment
+	err := x.Table("attachment").Where("issue_id > 0").Find(&issueAttachments)
 	assert.NoError(t, err)
 	for _, attach := range issueAttachments {
-		assert.Greater(t, attach.RepoID, 0)
-		assert.Greater(t, attach.IssueID, 0)
+		assert.Greater(t, attach.RepoID, int64(0))
+		assert.Greater(t, attach.IssueID, int64(0))
 		var issue Issue
 		has, err := x.ID(attach.IssueID).Get(&issue)
 		assert.NoError(t, err)
 		assert.True(t, has)
 		assert.EqualValues(t, attach.RepoID, issue.RepoID)
 	}
 
-	var releaseAttachments []*Attachment
-	err = x.Where("release_id > 0").Find(&releaseAttachments)
+	var releaseAttachments []*NewAttachment
+	err = x.Table("attachment").Where("release_id > 0").Find(&releaseAttachments)
 	assert.NoError(t, err)
 	for _, attach := range releaseAttachments {
-		assert.Greater(t, attach.RepoID, 0)
-		assert.Greater(t, attach.IssueID, 0)
+		assert.Greater(t, attach.RepoID, int64(0))
+		assert.Greater(t, attach.ReleaseID, int64(0))
 		var release Release
 		has, err := x.ID(attach.ReleaseID).Get(&release)
 		assert.NoError(t, err)

diff --git a/models/migrations/v1_22/v283.go b/models/migrations/v1_22/v283.go
@@ -4,7 +4,10 @@
 package v1_22 //nolint
 
 import (
+	"fmt"
+
 	"xorm.io/xorm"
+	"xorm.io/xorm/schemas"
 )
 
 func AddCombinedIndexToIssueUser(x *xorm.Engine) error {
@@ -20,8 +23,18 @@ func AddCombinedIndexToIssueUser(x *xorm.Engine) error {
 		return err
 	}
 	for _, issueUser := range duplicatedIssueUsers {
-		if _, err := x.Exec("delete from issue_user where id in (SELECT id FROM issue_user WHERE issue_id = ? and uid = ? limit ?)", issueUser.IssueID, issueUser.UID, issueUser.Cnt-1); err != nil {
-			return err
+		if x.Dialect().URI().DBType == schemas.MSSQL {
+			if _, err := x.Exec(fmt.Sprintf("delete from issue_user where id in (SELECT top %d id FROM issue_user WHERE issue_id = ? and uid = ?)", issueUser.Cnt-1), issueUser.IssueID, issueUser.UID); err != nil {
+				return err
+			}
+		} else {
+			var ids []int64
+			if err := x.SQL("SELECT id FROM issue_user WHERE issue_id = ? and uid = ? limit ?", issueUser.IssueID, issueUser.UID, issueUser.Cnt-1).Find(&ids); err != nil {
+				return err
+			}
+			if _, err := x.Table("issue_user").In("id", ids).Delete(); err != nil {
+				return err
+			}
 		}
 	}
 

diff --git a/models/migrations/v1_22/v286.go b/models/migrations/v1_22/v286.go
@@ -36,9 +36,9 @@ func expandHashReferencesToSha256(x *xorm.Engine) error {
 		if setting.Database.Type.IsMSSQL() {
 			// drop indexes that need to be re-created afterwards
 			droppedIndexes := []string{
-				"DROP INDEX commit_status.IDX_commit_status_context_hash",
-				"DROP INDEX review_state.UQE_review_state_pull_commit_user",
-				"DROP INDEX repo_archiver.UQE_repo_archiver_s",
+				"DROP INDEX IF EXISTS [IDX_commit_status_context_hash] ON [commit_status]",
+				"DROP INDEX IF EXISTS [UQE_review_state_pull_commit_user] ON [review_state]",
+				"DROP INDEX IF EXISTS [UQE_repo_archiver_s] ON [repo_archiver]",
 			}
 			for _, s := range droppedIndexes {
 				_, err := db.Exec(s)
@@ -53,7 +53,7 @@ func expandHashReferencesToSha256(x *xorm.Engine) error {
 			if setting.Database.Type.IsMySQL() {
 				_, err = db.Exec(fmt.Sprintf("ALTER TABLE `%s` MODIFY COLUMN `%s` VARCHAR(64)", alts[0], alts[1]))
 			} else if setting.Database.Type.IsMSSQL() {
-				_, err = db.Exec(fmt.Sprintf("ALTER TABLE `%s` ALTER COLUMN `%s` VARCHAR(64)", alts[0], alts[1]))
+				_, err = db.Exec(fmt.Sprintf("ALTER TABLE [%s] ALTER COLUMN [%s] VARCHAR(64)", alts[0], alts[1]))
 			} else {
 				_, err = db.Exec(fmt.Sprintf("ALTER TABLE `%s` ALTER COLUMN `%s` TYPE VARCHAR(64)", alts[0], alts[1]))
 			}