From ddc3738042eaad63fde553368cbd2a58acccb124 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Sun, 17 Jul 2022 15:22:05 +0300 Subject: [PATCH] Implement database changes and store methods for global end organization secrets --- go.mod | 4 +- go.sum | 10 ++-- server/model/secret.go | 18 ++++++-- server/plugins/secrets/builtin.go | 32 ++++++++++++- .../migration/006_secrets_add_user.go | 46 +++++++++++++++++++ server/store/datastore/migration/common.go | 36 +++++++++++++++ server/store/datastore/migration/migration.go | 1 + server/store/datastore/secret.go | 21 ++++++++- server/store/datastore/secret_test.go | 39 +++++++++++++++- server/store/store.go | 2 +- 10 files changed, 193 insertions(+), 16 deletions(-) create mode 100644 server/store/datastore/migration/006_secrets_add_user.go diff --git a/go.mod b/go.mod index eeb714cd122..a9c11594c5b 100644 --- a/go.mod +++ b/go.mod @@ -41,8 +41,8 @@ require ( google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - xorm.io/builder v0.3.10 - xorm.io/xorm v1.3.0 + xorm.io/builder v0.3.12 + xorm.io/xorm v1.3.1 ) require ( diff --git a/go.sum b/go.sum index a03573fa8f6..c70b092e0c5 100644 --- a/go.sum +++ b/go.sum @@ -1186,8 +1186,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -xorm.io/builder v0.3.9/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= -xorm.io/builder v0.3.10 h1:Rvkncad3Lo9YIVqCbgIf6QnpR/HcW3IEr0AANNpuyMQ= -xorm.io/builder v0.3.10/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= -xorm.io/xorm v1.3.0 h1:UsVke0wyAk3tJcb0j15gLWv2DEshVUnySVyvcYDny8w= -xorm.io/xorm v1.3.0/go.mod h1:cEaWjDPqoIusTkmDAG+krCcPcTglqo8CDU8geX/yhko= +xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/builder v0.3.12 h1:ASZYX7fQmy+o8UJdhlLHSW57JDOkM8DNhcAF5d0LiJM= +xorm.io/builder v0.3.12/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/xorm v1.3.1 h1:z5egKrDoOLqZFhMjcGF4FBHiTmE5/feQoHclfhNidfM= +xorm.io/xorm v1.3.1/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw= diff --git a/server/model/secret.go b/server/model/secret.go index c4834448803..373d6ec54b8 100644 --- a/server/model/secret.go +++ b/server/model/secret.go @@ -41,7 +41,7 @@ type SecretService interface { // SecretStore persists secret information to storage. type SecretStore interface { SecretFind(*Repo, string) (*Secret, error) - SecretList(*Repo) ([]*Secret, error) + SecretList(*Repo, bool) ([]*Secret, error) SecretCreate(*Secret) error SecretUpdate(*Secret) error SecretDelete(*Secret) error @@ -51,8 +51,9 @@ type SecretStore interface { // swagger:model registry type Secret struct { ID int64 `json:"id" xorm:"pk autoincr 'secret_id'"` - RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX 'secret_repo_id'"` - Name string `json:"name" xorm:"UNIQUE(s) INDEX 'secret_name'"` + Owner string `json:"-" xorm:"NOT NULL DEFAULT '' UNIQUE(s) INDEX 'secret_owner'"` + RepoID int64 `json:"-" xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_repo_id'"` + Name string `json:"name" xorm:"NOT NULL UNIQUE(s) INDEX 'secret_name'"` Value string `json:"value,omitempty" xorm:"TEXT 'secret_value'"` Images []string `json:"image" xorm:"json 'secret_images'"` Events []WebhookEvent `json:"event" xorm:"json 'secret_events'"` @@ -65,6 +66,16 @@ func (Secret) TableName() string { return "secrets" } +// Global secret. +func (s Secret) Global() bool { + return s.RepoID == 0 && s.Owner == "" +} + +// Organization secret. +func (s Secret) Organization() bool { + return s.RepoID == 0 && s.Owner != "" +} + // Match returns true if an image and event match the restricted list. func (s *Secret) Match(event WebhookEvent) bool { if len(s.Events) == 0 { @@ -119,6 +130,7 @@ func (s *Secret) Validate() error { func (s *Secret) Copy() *Secret { return &Secret{ ID: s.ID, + Owner: s.Owner, RepoID: s.RepoID, Name: s.Name, Images: s.Images, diff --git a/server/plugins/secrets/builtin.go b/server/plugins/secrets/builtin.go index fab49585d05..377dcecb9e7 100644 --- a/server/plugins/secrets/builtin.go +++ b/server/plugins/secrets/builtin.go @@ -21,11 +21,39 @@ func (b *builtin) SecretFind(repo *model.Repo, name string) (*model.Secret, erro } func (b *builtin) SecretList(repo *model.Repo) ([]*model.Secret, error) { - return b.store.SecretList(repo) + return b.store.SecretList(repo, false) } func (b *builtin) SecretListBuild(repo *model.Repo, build *model.Build) ([]*model.Secret, error) { - return b.store.SecretList(repo) + s, err := b.store.SecretList(repo, true) + if err != nil { + return nil, err + } + + // Return only secrets with unique name + // Priority order in case of duplicate names are repository, user/organization, global + secrets := make([]*model.Secret, 0, len(s)) + uniq := make(map[string]struct{}) + for _, cond := range []struct { + Global bool + Organization bool + }{ + {}, + {Organization: true}, + {Global: true}, + } { + for _, secret := range s { + if secret.Global() == cond.Global && secret.Organization() == cond.Organization { + continue + } + if _, ok := uniq[secret.Name]; ok { + continue + } + uniq[secret.Name] = struct{}{} + secrets = append(secrets, secret) + } + } + return secrets, nil } func (b *builtin) SecretCreate(repo *model.Repo, in *model.Secret) error { diff --git a/server/store/datastore/migration/006_secrets_add_user.go b/server/store/datastore/migration/006_secrets_add_user.go new file mode 100644 index 00000000000..0eca42f5441 --- /dev/null +++ b/server/store/datastore/migration/006_secrets_add_user.go @@ -0,0 +1,46 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migration + +import ( + "xorm.io/xorm" +) + +type SecretV006 struct { + Owner string `json:"-" xorm:"NOT NULL DEFAULT '' UNIQUE(s) INDEX 'secret_owner'"` + RepoID int64 `json:"-" xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_repo_id'"` + Name string `json:"name" xorm:"NOT NULL UNIQUE(s) INDEX 'secret_name'"` +} + +// TableName return database table name for xorm +func (SecretV006) TableName() string { + return "secrets" +} + +var alterTableSecretsAddUserCol = task{ + name: "alter-table-add-secrets-user-id", + fn: func(sess *xorm.Session) error { + if err := sess.Sync2(new(SecretV006)); err != nil { + return err + } + if err := alterColumnDefault(sess, "secrets", "secret_repo_id", "0"); err != nil { + return err + } + if err := alterColumnNull(sess, "secrets", "secret_repo_id", false); err != nil { + return err + } + return alterColumnNull(sess, "secrets", "secret_name", false) + }, +} diff --git a/server/store/datastore/migration/common.go b/server/store/datastore/migration/common.go index 000df760a28..c3b6dc903be 100644 --- a/server/store/datastore/migration/common.go +++ b/server/store/datastore/migration/common.go @@ -212,6 +212,42 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin return nil } +func alterColumnDefault(sess *xorm.Session, table, column, defValue string) error { + dialect := sess.Engine().Dialect().URI().DBType + switch dialect { + case schemas.MYSQL: + _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` COLUMN `%s` SET DEFAULT %s;", table, column, defValue)) + return err + case schemas.POSTGRES: + _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` ALTER COLUMN `%s` SET DEFAULT %s;", table, column, defValue)) + return err + case schemas.SQLITE: + return nil + default: + return fmt.Errorf("dialect '%s' not supported", dialect) + } +} + +func alterColumnNull(sess *xorm.Session, table, column string, null bool) error { + val := "NULL" + if !null { + val = "NOT NULL" + } + dialect := sess.Engine().Dialect().URI().DBType + switch dialect { + case schemas.MYSQL: + _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` COLUMN `%s` SET %s;", table, column, val)) + return err + case schemas.POSTGRES: + _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` ALTER COLUMN `%s` SET %s;", table, column, val)) + return err + case schemas.SQLITE: + return nil + default: + return fmt.Errorf("dialect '%s' not supported", dialect) + } +} + var ( whitespaces = regexp.MustCompile(`\s+`) columnSeparator = regexp.MustCompile(`\s?,\s?`) diff --git a/server/store/datastore/migration/migration.go b/server/store/datastore/migration/migration.go index 7c165c433cd..558992789c4 100644 --- a/server/store/datastore/migration/migration.go +++ b/server/store/datastore/migration/migration.go @@ -33,6 +33,7 @@ var migrationTasks = []*task{ &fixPRSecretEventName, &alterTableReposDropCounter, &dropSenders, + &alterTableSecretsAddUserCol, } var allBeans = []interface{}{ diff --git a/server/store/datastore/secret.go b/server/store/datastore/secret.go index d64ad0e27a4..c83ba0a1e2e 100644 --- a/server/store/datastore/secret.go +++ b/server/store/datastore/secret.go @@ -16,6 +16,8 @@ package datastore import ( "github.com/woodpecker-ci/woodpecker/server/model" + + "xorm.io/builder" ) func (s storage) SecretFind(repo *model.Repo, name string) (*model.Secret, error) { @@ -26,9 +28,24 @@ func (s storage) SecretFind(repo *model.Repo, name string) (*model.Secret, error return secret, wrapGet(s.engine.Get(secret)) } -func (s storage) SecretList(repo *model.Repo) ([]*model.Secret, error) { +func (s storage) GlobalSecretList() ([]*model.Secret, error) { + secrets := make([]*model.Secret, 0, perPage) + return secrets, s.engine.Where(builder.And(builder.Eq{"secret_owner": ""}, builder.Eq{"secret_repo_id": 0})).Find(&secrets) +} + +func (s storage) UserSecretList(owner string) ([]*model.Secret, error) { secrets := make([]*model.Secret, 0, perPage) - return secrets, s.engine.Where("secret_repo_id = ?", repo.ID).Find(&secrets) + return secrets, s.engine.Where("secret_owner = ?", owner).Find(&secrets) +} + +func (s storage) SecretList(repo *model.Repo, all bool) ([]*model.Secret, error) { + secrets := make([]*model.Secret, 0, perPage) + var cond builder.Cond = builder.Eq{"secret_repo_id": repo.ID} + if all { + cond = cond.Or(builder.Eq{"secret_owner": repo.Owner}). + Or(builder.And(builder.Eq{"secret_owner": ""}, builder.Eq{"secret_repo_id": 0})) + } + return secrets, s.engine.Where(cond).Find(&secrets) } func (s storage) SecretCreate(secret *model.Secret) error { diff --git a/server/store/datastore/secret_test.go b/server/store/datastore/secret_test.go index 0285a13cd73..998c4a1575c 100644 --- a/server/store/datastore/secret_test.go +++ b/server/store/datastore/secret_test.go @@ -80,12 +80,49 @@ func TestSecretList(t *testing.T) { Name: "baz", Value: "qux", })) + assert.NoError(t, store.SecretCreate(&model.Secret{ + Owner: "org", + Name: "usr", + Value: "sec", + })) + assert.NoError(t, store.SecretCreate(&model.Secret{ + Name: "global", + Value: "val", + })) - list, err := store.SecretList(&model.Repo{ID: 1}) + list, err := store.SecretList(&model.Repo{ID: 1, Owner: "org"}, false) assert.NoError(t, err) assert.Len(t, list, 2) } +func TestSecretBuildList(t *testing.T) { + store, closer := newTestStore(t, new(model.Secret)) + defer closer() + + assert.NoError(t, store.SecretCreate(&model.Secret{ + Owner: "org", + Name: "usr", + Value: "sec", + })) + assert.NoError(t, store.SecretCreate(&model.Secret{ + RepoID: 1, + Name: "foo", + Value: "bar", + })) + assert.NoError(t, store.SecretCreate(&model.Secret{ + RepoID: 1, + Name: "baz", + Value: "qux", + })) + assert.NoError(t, store.SecretCreate(&model.Secret{ + Name: "global", + Value: "val", + })) + list, err := store.SecretList(&model.Repo{ID: 1, Owner: "org"}, true) + assert.NoError(t, err) + assert.Len(t, list, 4) +} + func TestSecretUpdate(t *testing.T) { store, closer := newTestStore(t, new(model.Secret)) defer closer() diff --git a/server/store/store.go b/server/store/store.go index 2279983bf6d..e4471ed868a 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -106,7 +106,7 @@ type Store interface { // Secrets SecretFind(*model.Repo, string) (*model.Secret, error) - SecretList(*model.Repo) ([]*model.Secret, error) + SecretList(*model.Repo, bool) ([]*model.Secret, error) SecretCreate(*model.Secret) error SecretUpdate(*model.Secret) error SecretDelete(*model.Secret) error