Retrieve Secrets for secrets group

go.sum

@@ -4,10 +4,16 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cyberark/conjur-api-go v0.6.0 h1:QQYmFRhcCvmtZ9oSRoXCxWb7uRjppfu5lcEwo4HEjtg=
+github.com/cyberark/conjur-api-go v0.6.0/go.mod h1:uM96pLpckwYYAWRSbrsw+TT0y3kg49QCEGpdpa9dJ34=
 github.com/cyberark/conjur-api-go v0.8.0 h1:y7/l7tv92h8rmpnOkebYwz/KSqUWe+IrLVITBk3ZWVQ=
 github.com/cyberark/conjur-api-go v0.8.0/go.mod h1:HZ5RoBhAB2KwnxyXbQ29DwpviRVg7SMRq7QhwtFjN3Q=
 github.com/cyberark/conjur-authn-k8s-client v0.19.1 h1:/o7De4Br4p1j2p9gOPQuurkdjypiHlmg+k2GwoGd1ik=
 github.com/cyberark/conjur-authn-k8s-client v0.19.1/go.mod h1:tD6+rie3c7LFclihIzg12vVK6+yKm0NB+3+0Pmau/A4=
+github.com/cyberark/conjur-authn-k8s-client v0.22.0 h1:ZAY96+k0UR4ABp2QBcjkUCXgF/scNC+ItR19tAY9yKM=
+github.com/cyberark/conjur-authn-k8s-client v0.22.0/go.mod h1:tD6+rie3c7LFclihIzg12vVK6+yKm0NB+3+0Pmau/A4=
+github.com/cyberark/secrets-provider-for-k8s v1.1.5 h1:ntoCJ+lEopdbda2XO/B0bjoxBaGdZ+imdhNNSoVnqiA=
+github.com/cyberark/secrets-provider-for-k8s v1.1.5/go.mod h1:JvqaYzj+2XdzDVb6clEh0+ppzpj48oiMTW3UGGXGs6A=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,11 +44,14 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
 github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.2.2-0.20181110203027-b4936e06046b+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -68,8 +77,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac h1:wbW+Bybf9pXxnCFAOWZTqkRjAc7rAIwo2e1ArUhiHxg=
+github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -83,6 +96,7 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -108,6 +122,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180709060233-1b2967e3c290/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -117,6 +132,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -160,6 +176,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=

pkg/log/messages/error_messages.go

@@ -73,3 +73,4 @@ const CSPFK049E string = "CSPFK049E Failed to validate Pod annotations"
 // Push to File
 const CSPFK050E string = "CSPFK050E Failed to unmarshal Push-to-File secrets. Reason: %s"
 const CSPFK051E string = "CSPFK051E Unknown file format '%s'"
+const CSPFK052E string = "CSPFK052E Failed to retrieve secrets. Reason: %s"

pkg/secrets/clients/conjur/mocks/conjur_secrets_retriever.go

@@ -53,3 +53,10 @@ func NewConjurMockClient() ConjurMockClient {
 	}
 }
 
+func (c ConjurMockClient) AddSecret(
+	secrets map[string]string,
+) {
+	for id, secret := range secrets {
+		c.Database[id] = secret
+	}
+}

pkg/secrets/pushtofile/retrieve_secrets.go

@@ -0,0 +1,74 @@
+package pushtofile
+
+import (
+	"strings"
+
+	"github.com/cyberark/conjur-authn-k8s-client/pkg/access_token"
+	"github.com/cyberark/conjur-authn-k8s-client/pkg/log"
+	"github.com/cyberark/secrets-provider-for-k8s/pkg/log/messages"
+	"github.com/cyberark/secrets-provider-for-k8s/pkg/secrets/clients/conjur"
+)
+
+type secret struct {
+	Alias string
+	Value string
+}
+type fetcher interface {
+	SecretFetcher(secretIds []string) (map[string][]byte, error)
+}
+
+type conjurSecretFetch struct {
+	accessToken access_token.AccessToken
+}
+
+func (s conjurSecretFetch) SecretFetcher(secretIds []string) (map[string][]byte, error) {
+	accessTokenData, err := s.accessToken.Read()
+	if err != nil {
+		return nil, log.RecordedError(messages.CSPFK002E)
+	}
+	return conjur.RetrieveConjurSecrets(accessTokenData, secretIds)
+}
+
+// FetchSecretsForGroups parses the SecretsGroup, gets
+// the secrets from Conjur and updates the SecretsGroup with the secret
+func FetchSecretsForGroups(secretGroups *SecretGroups,
+	accessToken access_token.AccessToken) (map[string][]*secret, error) {
+	var conjurSecretFetcher = conjurSecretFetch{accessToken}
+	return fetchSecretsForGroups(conjurSecretFetcher, secretGroups)
+}
+
+func getAllIds(secretGroups *SecretGroups) []string {
+	ids := []string{}
+	for _, group := range *secretGroups {
+		for _, spec := range group.SecretSpecs {
+			ids = append(ids, spec.Id)
+		}
+	}
+	return ids
+}
+
+func fetchSecretsForGroups(secretsFetcherFunc fetcher,
+	secretGroups *SecretGroups,
+) (map[string][]*secret, error) {
+
+	secretsValues := map[string][]*secret{}
+
+	ids := getAllIds(secretGroups)
+	retrieved, err := secretsFetcherFunc.SecretFetcher(ids)
+	if err != nil {
+		return nil, log.RecordedError(messages.CSPFK052E, err.Error())
+	}
+	for _, group := range *secretGroups {
+		for _, spec := range group.SecretSpecs {
+			for id, retSecret := range retrieved {
+				if strings.Contains(id, spec.Id) {
+					fetchedSecret := new(secret)
+					fetchedSecret.Alias = spec.Alias
+					fetchedSecret.Value = string(retSecret)
+					secretsValues[group.Label] = append(secretsValues[group.Label], fetchedSecret)
+				}
+			}
+		}
+	}
+	return secretsValues, err
+}

pkg/secrets/pushtofile/retrieve_secrets_test.go

@@ -0,0 +1,137 @@
+package pushtofile
+
+import (
+	"testing"
+
+	conjurMocks "github.com/cyberark/secrets-provider-for-k8s/pkg/secrets/clients/conjur/mocks"
+	"github.com/stretchr/testify/assert"
+)
+
+type retrieveSecretsTestCase struct {
+	description string
+	secretSpecs map[string][]SecretSpec
+	assert      func(t *testing.T, result map[string][]*secret, err error)
+}
+
+func (tc retrieveSecretsTestCase) Run(t *testing.T, mockSecretFetch MockSecretFetch) {
+	t.Run(tc.description, func(t *testing.T) {
+		s := createSecretGroups(tc.secretSpecs)
+		ret, err := fetchSecretsForGroups(mockSecretFetch, &s)
+		tc.assert(t, ret, err)
+	})
+}
+
+func createSecretGroups(groupSpecs map[string][]SecretSpec) SecretGroups {
+	secretGroups := SecretGroups{}
+	for label, secretSpecs := range groupSpecs {
+		secretGroup := SecretGroup{
+			Label:       label,
+			SecretSpecs: secretSpecs,
+		}
+		secretGroups = append(secretGroups, secretGroup)
+	}
+	return secretGroups
+}
+
+func findGroupValues(group map[string][]*secret, label string) []*secret {
+	for key, secretGroup := range group {
+		if key == label {
+			return secretGroup
+		}
+	}
+	return nil
+}
+
+func assertGoodResults(expectedGroupValues map[string][]*secret) func(*testing.T, map[string][]*secret, error) {
+	return func(t *testing.T, result map[string][]*secret, err error) {
+
+		if !assert.NoError(t, err) {
+			return
+		}
+		for groupLabel, expValues := range expectedGroupValues {
+			actualValues := findGroupValues(result, groupLabel)
+			assert.NotNil(t, actualValues)
+			assert.True(t, assert.EqualValues(t, actualValues, expValues))
+		}
+	}
+}
+
+var retrieveSecretsTestCases = []retrieveSecretsTestCase{
+	{
+		description: "Happy Case",
+		secretSpecs: map[string][]SecretSpec{
+			"cache": []SecretSpec{
+				{Alias: "api-url", Id: "dev/openshift/api-url"},
+				{Alias: "username", Id: "dev/openshift/username"},
+				{Alias: "password", Id: "dev/openshift/password"},
+			},
+			"db": []SecretSpec{
+				{Alias: "api-url", Id: "ci/openshift/api-url"},
+				{Alias: "username", Id: "ci/openshift/username"},
+				{Alias: "password", Id: "ci/openshift/password"},
+			},
+		},
+		assert: assertGoodResults(map[string][]*secret{
+			"cache": []*secret{
+				{Alias: "api-url", Value: "https://postgres.example.com"},
+				{Alias: "username", Value: "admin"},
+				{Alias: "password", Value: "open-$e$ame"},
+			},
+			"db": []*secret{
+				{Alias: "api-url", Value: "https://ci.postgres.example.com"},
+				{Alias: "username", Value: "administrator"},
+				{Alias: "password", Value: "open-$e$ame"},
+			},
+		}),
+	},
+	{
+		description: "Bad ID",
+		secretSpecs: map[string][]SecretSpec{
+			"cache": []SecretSpec{
+				{Alias: "api-url", Id: "foo/openshift/bar"},
+				{Alias: "username", Id: "dev/openshift/username"},
+				{Alias: "password", Id: "dev/openshift/password"},
+			},
+			"db": []SecretSpec{
+				{Alias: "api-url", Id: "ci/openshift/api-url"},
+				{Alias: "username", Id: "ci/openshift/username"},
+				{Alias: "password", Id: "ci/openshift/password"},
+			},
+		},
+		assert: func(t *testing.T, result map[string][]*secret, err error) {
+			assert.Contains(t, err.Error(), "Failed to retrieve secrets")
+		},
+	},
+}
+
+type MockSecretFetch struct {
+	accessToken      conjurMocks.MockAccessToken
+	conjurMockClient conjurMocks.ConjurMockClient
+}
+
+func (s MockSecretFetch) SecretFetcher(secretIds []string) (map[string][]byte, error) {
+
+	accessTokenData, _ := s.accessToken.Read()
+	return s.conjurMockClient.RetrieveSecrets(accessTokenData, secretIds)
+}
+
+func mockInit(s *MockSecretFetch) {
+	s.conjurMockClient = conjurMocks.NewConjurMockClient()
+	mockSecrets := map[string]string{
+		"dev/openshift/api-url":  "https://postgres.example.com",
+		"dev/openshift/username": "admin",
+		"dev/openshift/password": "open-$e$ame",
+		"ci/openshift/api-url":   "https://ci.postgres.example.com",
+		"ci/openshift/username":  "administrator",
+		"ci/openshift/password":  "open-$e$ame",
+	}
+	s.conjurMockClient.AddSecret(mockSecrets)
+}
+
+func TestRetrieveSecrets(t *testing.T) {
+	var mockSecretFetch MockSecretFetch
+	mockInit(&mockSecretFetch)
+	for _, tc := range retrieveSecretsTestCases {
+		tc.Run(t, mockSecretFetch)
+	}
+}