diff --git a/pkg/agent/cache/apiserviceinstance.go b/pkg/agent/cache/apiserviceinstance.go index 102b5a1b4..106cd1d39 100644 --- a/pkg/agent/cache/apiserviceinstance.go +++ b/pkg/agent/cache/apiserviceinstance.go @@ -10,7 +10,7 @@ import ( func (c *cacheManager) AddAPIServiceInstance(resource *v1.ResourceInstance) { defer c.setCacheUpdated(true) - c.instanceMap.Set(resource.Metadata.ID, resource) + c.instanceMap.SetWithSecondaryKey(resource.Metadata.ID, resource.Name, resource) } // GetAPIServiceInstanceKeys - returns keys for APIServiceInstance cache @@ -36,6 +36,21 @@ func (c *cacheManager) GetAPIServiceInstanceByID(id string) (*v1.ResourceInstanc return nil, err } +// GetAPIServiceInstanceByName - returns resource from APIServiceInstance cache based on instance name +func (c *cacheManager) GetAPIServiceInstanceByName(name string) (*v1.ResourceInstance, error) { + c.ApplyResourceReadLock() + defer c.ReleaseResourceReadLock() + + item, err := c.instanceMap.GetBySecondaryKey(name) + if item != nil { + instance, ok := item.(*v1.ResourceInstance) + if ok { + return instance, nil + } + } + return nil, err +} + // DeleteAPIServiceInstance - remove APIServiceInstance resource from cache based on instance ID func (c *cacheManager) DeleteAPIServiceInstance(id string) error { defer c.setCacheUpdated(true) diff --git a/pkg/agent/cache/manager.go b/pkg/agent/cache/manager.go index 16ae08464..6da3494c5 100644 --- a/pkg/agent/cache/manager.go +++ b/pkg/agent/cache/manager.go @@ -38,6 +38,7 @@ type Manager interface { AddAPIServiceInstance(resource *v1.ResourceInstance) GetAPIServiceInstanceKeys() []string GetAPIServiceInstanceByID(id string) (*v1.ResourceInstance, error) + GetAPIServiceInstanceByName(apiName string) (*v1.ResourceInstance, error) DeleteAPIServiceInstance(id string) error DeleteAllAPIServiceInstance() diff --git a/pkg/agent/eventsync.go b/pkg/agent/eventsync.go index 261335b6d..66278ca59 100644 --- a/pkg/agent/eventsync.go +++ b/pkg/agent/eventsync.go @@ -35,7 +35,8 @@ func NewEventSync() (*EventSync, error) { // add attribute migration to migrations attributeMigration := migrate.NewAttributeMigration(agent.apicClient, agent.cfg) ardMigration := migrate.NewArdMigration(agent.apicClient, agent.cfg) - migrations = append(migrations, attributeMigration, ardMigration) + apisiMigration := migrate.NewAPISIMigration(agent.apicClient, agent.cfg) + migrations = append(migrations, attributeMigration, ardMigration, apisiMigration) if isMpEnabled { // add marketplace migration to migrations diff --git a/pkg/apic/apiservice.go b/pkg/apic/apiservice.go index c365198d3..ca6ced763 100644 --- a/pkg/apic/apiservice.go +++ b/pkg/apic/apiservice.go @@ -53,11 +53,21 @@ func (c *ServiceClient) buildAPIService(serviceBody *ServiceBody) *mv1a.APIServi } svcDetails := buildAgentDetailsSubResource(serviceBody, true, serviceBody.ServiceAgentDetails) + c.setMigrationFlags(svcDetails) + util.SetAgentDetails(svc, svcDetails) return svc } +func (c *ServiceClient) setMigrationFlags(svcDetails map[string]interface{}) { + svcDetails[defs.InstanceMigration] = defs.MigrationCompleted + + if c.cfg.IsMarketplaceSubsEnabled() { + svcDetails[defs.MarketplaceMigration] = defs.MigrationCompleted + } +} + func (c *ServiceClient) getOwnerObject(serviceBody *ServiceBody, warning bool) (*v1.Owner, error) { if id, found := c.getTeamFromCache(serviceBody.TeamName); found { return &v1.Owner{ diff --git a/pkg/apic/apiservice_test.go b/pkg/apic/apiservice_test.go index 543153208..517eeec68 100644 --- a/pkg/apic/apiservice_test.go +++ b/pkg/apic/apiservice_test.go @@ -105,7 +105,6 @@ func TestCreateService(t *testing.T) { apiSvc, err := client.PublishService(&cloneServiceBody) assert.Nil(t, err) assert.NotNil(t, apiSvc) - assert.Equal(t, &cloneServiceBody.serviceContext.revisionName, &cloneServiceBody.serviceContext.instanceName) // this should fail httpClient.SetResponses([]api.MockResponse{ { @@ -274,6 +273,10 @@ func TestUpdateService(t *testing.T) { FileName: "./testdata/apiservice.json", // for call to update the service subresource RespCode: http.StatusOK, }, + { + FileName: "./testdata/apiservice.json", // for call to update the service subresource + RespCode: http.StatusOK, + }, { FileName: "./testdata/servicerevision.json", // for call to update the serviceRevision RespCode: http.StatusOK, @@ -309,7 +312,6 @@ func TestUpdateService(t *testing.T) { apiSvc, err := client.PublishService(&cloneServiceBody) assert.Nil(t, err) assert.NotNil(t, apiSvc) - assert.Equal(t, &cloneServiceBody.serviceContext.revisionName, &cloneServiceBody.serviceContext.instanceName) // tests for updating existing instance with same endpoint httpClient.SetResponses([]api.MockResponse{ @@ -321,6 +323,10 @@ func TestUpdateService(t *testing.T) { FileName: "./testdata/apiservice.json", // for call to update the service subresource RespCode: http.StatusOK, }, + { + FileName: "./testdata/apiservice.json", // for call to update the service subresource + RespCode: http.StatusOK, + }, { FileName: "./testdata/servicerevision.json", // for call to update the serviceRevision RespCode: http.StatusOK, diff --git a/pkg/apic/apiserviceinstance.go b/pkg/apic/apiserviceinstance.go index d7fda0ef2..0b1c4cc22 100644 --- a/pkg/apic/apiserviceinstance.go +++ b/pkg/apic/apiserviceinstance.go @@ -3,7 +3,6 @@ package apic import ( "encoding/json" "errors" - "fmt" "net/http" "strconv" @@ -141,37 +140,24 @@ func (c *ServiceClient) processInstance(serviceBody *ServiceBody) error { return err } - var httpMethod string - var instance *mv1a.APIServiceInstance + // creating new instance + instance := c.buildAPIServiceInstance(serviceBody, getRevisionPrefix(serviceBody), endpoints) - instanceURL := c.cfg.GetInstancesURL() - instancePrefix := getRevisionPrefix(serviceBody) - instanceName := instancePrefix + "." + strconv.Itoa(serviceBody.serviceContext.revisionCount) - - if serviceBody.serviceContext.revisionAction == addAPI { - httpMethod = http.MethodPost - instance = c.buildAPIServiceInstance(serviceBody, instanceName, endpoints) - } - - if serviceBody.serviceContext.revisionAction == updateAPI { - httpMethod = http.MethodPut - instances, err := c.getRevisionInstances(instanceName, instanceURL) + if serviceBody.serviceContext.serviceAction == updateAPI { + prevInst, err := c.getLastInstance(serviceBody, c.createAPIServerURL(instance.GetKindLink())) if err != nil { return err } - if len(instances) == 0 { - return fmt.Errorf("no instance found named '%s' for revision '%s'", instanceName, serviceBody.serviceContext.revisionName) + + if prevInst != nil { + // updating existing instance + instance = c.updateAPIServiceInstance(serviceBody, prevInst, endpoints) } - instanceURL = instanceURL + "/" + instanceName - instance = c.updateAPIServiceInstance(serviceBody, instances[0], endpoints) } - buffer, err := json.Marshal(instance) - if err != nil { - return err - } + addSpecHashToResource(instance) - ri, err := c.executeAPIServiceAPI(httpMethod, instanceURL, buffer) + ri, err := c.CreateOrUpdateResource(instance) if err != nil { if serviceBody.serviceContext.serviceAction == addAPI { _, rollbackErr := c.rollbackAPIService(serviceBody.serviceContext.serviceName) @@ -199,7 +185,7 @@ func (c *ServiceClient) processInstance(serviceBody *ServiceBody) error { } c.caches.AddAPIServiceInstance(ri) - serviceBody.serviceContext.instanceName = instanceName + serviceBody.serviceContext.instanceName = instance.Name return err } @@ -233,13 +219,23 @@ func createInstanceEndpoint(endpoints []EndpointDefinition) ([]mv1a.ApiServiceIn return endPoints, nil } -func (c *ServiceClient) getRevisionInstances(name, url string) ([]*mv1a.APIServiceInstance, error) { - // Check if instances exist for the current revision. - queryParams := map[string]string{ - "query": "name==" + name, - } +func (c *ServiceClient) getLastInstance(serviceBody *ServiceBody, url string) (*mv1a.APIServiceInstance, error) { + // start from latest revision, find first instance + for i := serviceBody.serviceContext.revisionCount; i > 0; i-- { + queryParams := map[string]string{ + "query": "metadata.references.name==" + getRevisionPrefix(serviceBody) + "." + strconv.Itoa(i), + } - return c.GetAPIServiceInstances(queryParams, url) + instances, err := c.GetAPIServiceInstances(queryParams, url) + if err != nil { + return nil, err + } + + if len(instances) > 0 { + return instances[0], nil + } + } + return nil, nil } // GetAPIServiceInstanceByName - Returns the API service instance for specified name diff --git a/pkg/apic/client.go b/pkg/apic/client.go index 946d0211c..549e656e8 100644 --- a/pkg/apic/client.go +++ b/pkg/apic/client.go @@ -806,6 +806,8 @@ func (c *ServiceClient) updateSpecORCreateResourceInstance(data *v1.ResourceInst existingRI, err = c.caches.GetAccessRequestDefinitionByName(data.Name) case mv1a.CredentialRequestDefinitionGVK().Kind: existingRI, err = c.caches.GetCredentialRequestDefinitionByName(data.Name) + case mv1a.APIServiceInstanceGVK().Kind: + existingRI, err = c.caches.GetAPIServiceInstanceByName(data.Name) } if err == nil && existingRI != nil { diff --git a/pkg/apic/definitions/definitions.go b/pkg/apic/definitions/definitions.go index 5c755c782..47f29d0bc 100644 --- a/pkg/apic/definitions/definitions.go +++ b/pkg/apic/definitions/definitions.go @@ -37,6 +37,7 @@ const ( ReferencesSubResource = "references" Subscription = "Subscription" MarketplaceMigration = "marketplace-migration" + InstanceMigration = "instance-migration" ) // market place provisioning migration diff --git a/pkg/apic/util.go b/pkg/apic/util.go new file mode 100644 index 000000000..80563b88a --- /dev/null +++ b/pkg/apic/util.go @@ -0,0 +1,24 @@ +package apic + +import ( + "fmt" + + v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" + "github.com/Axway/agent-sdk/pkg/apic/definitions" + "github.com/Axway/agent-sdk/pkg/util" +) + +func addSpecHashToResource(h v1.Interface) error { + ri, err := h.AsInstance() + if err != nil { + return err + } + + hashInt, err := util.ComputeHash(ri.Spec) + if err != nil { + return err + } + + util.SetAgentDetailsKey(h, definitions.AttrSpecHash, fmt.Sprintf("%v", hashInt)) + return nil +} diff --git a/pkg/cmd/cmd_test.go b/pkg/cmd/cmd_test.go index d2fc53fb1..0a7935a06 100644 --- a/pkg/cmd/cmd_test.go +++ b/pkg/cmd/cmd_test.go @@ -115,6 +115,7 @@ func TestRootCmdFlags(t *testing.T) { assertStringSliceCmdFlag(t, rootCmd, "central.ssl.cipherSuites", "centralSslCipherSuites", corecfg.TLSDefaultCipherSuitesStringSlice(), "List of supported cipher suites, comma separated") assertStringCmdFlag(t, rootCmd, "central.ssl.minVersion", "centralSslMinVersion", corecfg.TLSDefaultMinVersionString(), "Minimum acceptable SSL/TLS protocol version") assertStringCmdFlag(t, rootCmd, "central.ssl.maxVersion", "centralSslMaxVersion", "0", "Maximum acceptable SSL/TLS protocol version") + assertBooleanCmdFlag(t, rootCmd, "central.migration.cleanInstances", "centralMigrationCleanInstances", false, "Set this to clean all but latest instance, per stage, within an API Service") // Traceability Agent rootCmd = NewRootCmd("Test", "TestRootCmd", nil, nil, corecfg.TraceabilityAgent) diff --git a/pkg/config/centralconfig.go b/pkg/config/centralconfig.go index 04cfc6953..6777d8962 100644 --- a/pkg/config/centralconfig.go +++ b/pkg/config/centralconfig.go @@ -124,6 +124,7 @@ type CentralConfig interface { SetIsMarketplaceSubsEnabled(enabled bool) IsMarketplaceSubsEnabled() bool GetSingleURL() string + GetMigrationSettings() MigrationConfig } // CentralConfiguration - Structure to hold the central config @@ -143,6 +144,7 @@ type CentralConfiguration struct { APIServerVersion string `config:"apiServerVersion"` TagsToPublish string `config:"additionalTags"` AppendEnvironmentToTitle bool `config:"appendEnvironmentToTitle"` + MigrationSettings MigrationConfig `config:"migration"` Auth AuthConfig `config:"auth"` TLS TLSConfig `config:"ssl"` PollInterval time.Duration `config:"pollInterval"` @@ -202,6 +204,7 @@ func NewCentralConfig(agentType AgentType) CentralConfig { PageSize: 20, }, }, + MigrationSettings: newMigrationConfig(), } } @@ -212,6 +215,7 @@ func NewTestCentralConfig(agentType AgentType) CentralConfig { config.URL = "https://central.com" config.Environment = "environment" config.Auth = newTestAuthConfig() + config.MigrationSettings = newTestMigrationConfig() return config } @@ -430,6 +434,11 @@ func (c *CentralConfiguration) GetAuthConfig() AuthConfig { return c.Auth } +// GetMigrationSettings - Returns the Migration Config +func (c *CentralConfiguration) GetMigrationSettings() MigrationConfig { + return c.MigrationSettings +} + // GetTLSConfig - Returns the TLS Config func (c *CentralConfiguration) GetTLSConfig() TLSConfig { return c.TLS @@ -754,6 +763,7 @@ func AddCentralConfigProperties(props properties.Properties, agentType AgentType props.AddStringProperty(pathAdditionalTags, "", "Additional Tags to Add to discovered APIs when publishing to Amplify Central") props.AddBoolProperty(pathAppendEnvironmentToTitle, true, "When true API titles and descriptions will be appended with environment name") AddSubscriptionConfigProperties(props) + AddMigrationConfigProperties(props) } } @@ -832,9 +842,11 @@ func ParseCentralConfig(props properties.Properties, agentType AgentType) (Centr // set the notifications subscriptionConfig := ParseSubscriptionConfig(props) cfg.SubscriptionConfiguration = subscriptionConfig + cfg.MigrationSettings = ParseMigrationConfig(props) } return cfg, nil } + func supportsTraceability(agentType AgentType) bool { return agentType == TraceabilityAgent } diff --git a/pkg/config/migrationconfig.go b/pkg/config/migrationconfig.go new file mode 100644 index 000000000..feb3f1736 --- /dev/null +++ b/pkg/config/migrationconfig.go @@ -0,0 +1,53 @@ +package config + +import "github.com/Axway/agent-sdk/pkg/cmd/properties" + +// MigrationConfig - Interface for migration settings config +type MigrationConfig interface { + ShouldCleanInstances() bool + validate() +} + +// MigrationSettings - +type MigrationSettings struct { + CleanInstances bool +} + +func newMigrationConfig() MigrationConfig { + return &MigrationSettings{ + CleanInstances: false, + } +} + +func newTestMigrationConfig() MigrationConfig { + return &MigrationSettings{ + CleanInstances: true, + } +} + +func (m *MigrationSettings) validate() { +} + +// ShouldCleanInstances - returns the value fo CleanInstances +func (m *MigrationSettings) ShouldCleanInstances() bool { + return m.CleanInstances +} + +const ( + pathCleanInstances = "central.migration.cleanInstances" +) + +// AddMigrationConfigProperties - Adds the command properties needed for Migration Config +func AddMigrationConfigProperties(props properties.Properties) { + props.AddBoolProperty(pathCleanInstances, false, "Set this to clean all but latest instance, per stage, within an API Service") +} + +// ParseMigrationConfig - Parses the Migration Config values from the command line +func ParseMigrationConfig(props properties.Properties) MigrationConfig { + migrationConfig := newMigrationConfig() + migrationSettings := migrationConfig.(*MigrationSettings) + + migrationSettings.CleanInstances = props.BoolPropertyValue(pathCleanInstances) + + return migrationSettings +} diff --git a/pkg/config/migrationconfig_test.go b/pkg/config/migrationconfig_test.go new file mode 100644 index 000000000..e71283a5f --- /dev/null +++ b/pkg/config/migrationconfig_test.go @@ -0,0 +1,13 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMigrationConfig(t *testing.T) { + defConf := newMigrationConfig() + + assert.False(t, defConf.ShouldCleanInstances()) +} diff --git a/pkg/migrate/apisimigration.go b/pkg/migrate/apisimigration.go new file mode 100644 index 000000000..8a4c68341 --- /dev/null +++ b/pkg/migrate/apisimigration.go @@ -0,0 +1,179 @@ +package migrate + +import ( + "regexp" + "strconv" + "sync" + + v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" + mv1a "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/management/v1alpha1" + "github.com/Axway/agent-sdk/pkg/apic/definitions" + "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agent-sdk/pkg/util" + "github.com/Axway/agent-sdk/pkg/util/log" +) + +// APISIMigration - used for migrating API Service Instances +type APISIMigration struct { + migration + logger log.FieldLogger +} + +// NewAPISIMigration creates a new APISIMigration +func NewAPISIMigration(client client, cfg config.CentralConfig) *APISIMigration { + logger := log.NewFieldLogger(). + WithPackage("sdk.migrate"). + WithComponent("instance-migration") + + return &APISIMigration{ + migration: migration{ + client: client, + cfg: cfg, + }, + logger: logger, + } +} + +// Migrate checks an APIServiceInstance for the "scopes" key in the schema, and removes it if it is found. +func (m *APISIMigration) Migrate(ri *v1.ResourceInstance) (*v1.ResourceInstance, error) { + if ri.Kind != mv1a.APIServiceGVK().Kind { + return ri, nil + } + + // skip migration if instance migration is not enabled + if !m.cfg.GetMigrationSettings().ShouldCleanInstances() { + return ri, nil + } + + logger := m.logger.WithField(serviceName, ri.Name) + + if isMigrationCompleted(ri, definitions.InstanceMigration) { + // migration ran already + logger.Debugf("service instance migration already completed") + return ri, nil + } + + // get all revisions for this service + revisions, err := m.getRevisions(ri) + if err != nil { + return ri, err + } + logger.WithField("revisions", revisions).Debug("all revisions") + + // get all instances for each revision + wg := &sync.WaitGroup{} + errCh := make(chan error, len(revisions)) + instances := []*v1.ResourceInstance{} + instancesLock := sync.RWMutex{} + + for _, rev := range revisions { + wg.Add(1) + + go func(r *v1.ResourceInstance) { + defer wg.Done() + + revisionInstances, err := m.getInstances(r) + + instancesLock.Lock() + defer instancesLock.Unlock() + instances = append(instances, revisionInstances...) + + errCh <- err + }(rev) + } + + wg.Wait() + close(errCh) + + for e := range errCh { + if e != nil { + return ri, e + } + } + logger.WithField("instances", instances).Debug("all instances") + + err = m.cleanInstances(logger, instances) + if err != nil { + return ri, err + } + + // mark the migration as complete + util.SetAgentDetailsKey(ri, definitions.InstanceMigration, definitions.MigrationCompleted) + err = m.updateRI(ri) + return ri, err +} + +// updateRev gets a list of revisions for the service +func (m *APISIMigration) getRevisions(ri *v1.ResourceInstance) ([]*v1.ResourceInstance, error) { + url := m.cfg.GetRevisionsURL() + q := map[string]string{ + "query": queryFunc(ri.Name), + } + + return m.getAllRI(url, q) +} + +// updateRev gets a list of revisions for the service +func (m *APISIMigration) getInstances(ri *v1.ResourceInstance) ([]*v1.ResourceInstance, error) { + url := m.cfg.GetInstancesURL() + q := map[string]string{ + "query": queryFunc(ri.Name), + } + + return m.getAllRI(url, q) +} + +func (m *APISIMigration) cleanInstances(logger log.FieldLogger, instances []*v1.ResourceInstance) error { + logger.Tracef("cleaning instances") + type instanceNameIndex struct { + ri *v1.ResourceInstance + index int + } + + re := regexp.MustCompile(`([-\.a-z0-9]*)\.([0-9]*$)`) + + // sort all instances into buckets based on name, removing any index number, noting the highest + toKeep := map[string]instanceNameIndex{} + for _, inst := range instances { + iLog := logger.WithField(instanceName, inst.Name) + iLog.Tracef("handling instances") + name := inst.Name + result := re.FindAllStringSubmatch(name, -1) + group := name + index := 0 + var err error + if len(result) > 0 { + group = result[0][1] + index, err = strconv.Atoi(result[0][2]) + if err != nil { + return err + } + } + iLog = iLog.WithField("service-group", group).WithField("instance-index", index) + iLog.Tracef("parsed instance name") + + keepIndex := -1 + if i, ok := toKeep[group]; ok { + keepIndex = i.index + } + + thisNameIndex := instanceNameIndex{ + ri: inst, + index: index, + } + + if keepIndex == -1 { + iLog.Trace("first instance in group") + toKeep[group] = thisNameIndex + } else if keepIndex < index { + iLog.Tracef("removing previous high instance with name: %s", toKeep[group].ri.Name) + m.client.DeleteResourceInstance(toKeep[group].ri) + toKeep[group] = thisNameIndex + } else { + iLog.Tracef("removing this instance") + m.client.DeleteResourceInstance(thisNameIndex.ri) + } + } + + return nil +} diff --git a/pkg/migrate/apisimigration_test.go b/pkg/migrate/apisimigration_test.go new file mode 100644 index 000000000..8e32bcb3b --- /dev/null +++ b/pkg/migrate/apisimigration_test.go @@ -0,0 +1,173 @@ +package migrate + +import ( + "fmt" + "math/rand" + "strings" + "sync" + "testing" + + apiv1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" + v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" + mv1a "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/management/v1alpha1" + "github.com/Axway/agent-sdk/pkg/apic/definitions" + "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agent-sdk/pkg/util" + "github.com/stretchr/testify/assert" +) + +var defEnvName = config.NewTestCentralConfig(config.DiscoveryAgent).GetEnvironmentName() + +func createRevisionsResponse(serviceName string, numRevs int) []*v1.ResourceInstance { + revs := []*v1.ResourceInstance{} + for i := 1; i <= numRevs; i++ { + rev := mv1a.NewAPIServiceRevision(fmt.Sprintf("%v.%v", serviceName, i), defEnvName) + rev.Spec.ApiService = serviceName + + ri, _ := rev.AsInstance() + revs = append(revs, ri) + } + return revs +} + +func createInstanceResponse(serviceName string, numRevs int) []*v1.ResourceInstance { + insts := []*v1.ResourceInstance{} + for i := 1; i <= numRevs; i++ { + inst := mv1a.NewAPIServiceInstance(fmt.Sprintf("%v.%v", serviceName, i), defEnvName) + inst.Spec.ApiServiceRevision = fmt.Sprintf("%v.%v", serviceName, i) + + ri, _ := inst.AsInstance() + insts = append(insts, ri) + } + + rand.Shuffle(len(insts), func(i, j int) { + insts[i], insts[j] = insts[j], insts[i] + }) + + return insts +} + +func TestAPISIMigration(t *testing.T) { + tests := []struct { + name string + resource v1.Interface + expectErr bool + turnOff bool + migrationComplete bool + setMigCompelete bool + revisions []*v1.ResourceInstance + instances []*v1.ResourceInstance + expectedDeletes int + }{ + { + name: "called with non-apiservice returns without error", + resource: mv1a.NewAccessRequestDefinition("asdf", defEnvName), + }, + { + name: "called with apiservice and config off returns without error", + resource: mv1a.NewAPIService("asdf", defEnvName), + turnOff: true, + }, + { + name: "called with apiservice that previously was migrated", + resource: mv1a.NewAPIService("asdf", defEnvName), + setMigCompelete: true, + migrationComplete: true, + }, + { + name: "called with apiservice and with no revisions returns without error", + resource: mv1a.NewAPIService("asdf", defEnvName), + migrationComplete: true, + }, + { + name: "called with apiservice, revisions, and instances of same stage returns without error", + resource: mv1a.NewAPIService("apisi", defEnvName), + revisions: createRevisionsResponse("apisi", 10), + instances: createInstanceResponse("apisi", 10), + expectedDeletes: 9, + migrationComplete: true, + }, + { + name: "called with apiservice, revisions, and instances of diff stages returns without error", + resource: mv1a.NewAPIService("apisi", defEnvName), + revisions: append(createRevisionsResponse("apisi-stage1", 5), createRevisionsResponse("apisi-stage2", 5)...), + instances: append(createInstanceResponse("apisi-stage1", 5), createInstanceResponse("apisi-stage2", 5)...), + expectedDeletes: 8, + migrationComplete: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &mockAPISIMigClient{ + revisions: tt.revisions, + instances: tt.instances, + } + + cfg := config.NewTestCentralConfig(config.DiscoveryAgent) + cfg.(*config.CentralConfiguration).MigrationSettings.(*config.MigrationSettings).CleanInstances = !tt.turnOff + mig := NewAPISIMigration(c, cfg) + + resInst, _ := tt.resource.AsInstance() + if tt.setMigCompelete { + util.SetAgentDetailsKey(resInst, definitions.InstanceMigration, definitions.MigrationCompleted) + } + ri, err := mig.Migrate(resInst) + if tt.expectErr { + assert.NotNil(t, err) + return + } + + assert.Nil(t, err) + migVal, _ := util.GetAgentDetailsValue(ri, definitions.InstanceMigration) + if tt.migrationComplete { + assert.Equal(t, definitions.MigrationCompleted, migVal) + } else { + assert.Equal(t, "", migVal) + } + assert.Equal(t, tt.expectedDeletes, c.deleteCalls) + }) + } +} + +type mockAPISIMigClient struct { + sync.Mutex + deleteCalls int + revisions []*v1.ResourceInstance + instances []*v1.ResourceInstance + instanceReturned bool +} + +func (m *mockAPISIMigClient) ExecuteAPI(method, url string, queryParam map[string]string, buffer []byte) ([]byte, error) { + return nil, nil +} + +func (m *mockAPISIMigClient) GetAPIV1ResourceInstancesWithPageSize(query map[string]string, url string, pageSize int) ([]*apiv1.ResourceInstance, error) { + m.Lock() + defer m.Unlock() + if m.instanceReturned { + return nil, nil + } else if strings.Contains(url, "instances") { + m.instanceReturned = true + return m.instances, nil + } + return m.revisions, nil +} + +func (m *mockAPISIMigClient) UpdateResourceInstance(ri apiv1.Interface) (*apiv1.ResourceInstance, error) { + r, err := ri.AsInstance() + return r, err +} + +func (m *mockAPISIMigClient) CreateOrUpdateResource(data apiv1.Interface) (*apiv1.ResourceInstance, error) { + return nil, nil +} + +func (m *mockAPISIMigClient) CreateSubResource(rm apiv1.ResourceMeta, subs map[string]interface{}) error { + return nil +} + +func (m *mockAPISIMigClient) DeleteResourceInstance(ri apiv1.Interface) error { + m.deleteCalls++ + return nil +} diff --git a/pkg/migrate/ardmigration.go b/pkg/migrate/ardmigration.go index 01ad1eb59..99121ee8b 100644 --- a/pkg/migrate/ardmigration.go +++ b/pkg/migrate/ardmigration.go @@ -8,15 +8,16 @@ import ( // ArdMigration - used for migrating access request definitions type ArdMigration struct { - client client - cfg config.CentralConfig + migration } // NewArdMigration creates a new ArdMigration func NewArdMigration(client client, cfg config.CentralConfig) *ArdMigration { return &ArdMigration{ - client: client, - cfg: cfg, + migration: migration{ + client: client, + cfg: cfg, + }, } } diff --git a/pkg/migrate/ardmigration_test.go b/pkg/migrate/ardmigration_test.go index 7667dd0ca..8d04242cd 100644 --- a/pkg/migrate/ardmigration_test.go +++ b/pkg/migrate/ardmigration_test.go @@ -51,3 +51,7 @@ func (m mockArdMigClient) CreateOrUpdateResource(data apiv1.Interface) (*apiv1.R func (m mockArdMigClient) CreateSubResource(rm apiv1.ResourceMeta, subs map[string]interface{}) error { return nil } + +func (m mockArdMigClient) DeleteResourceInstance(ri apiv1.Interface) error { + return nil +} diff --git a/pkg/migrate/attributemigration.go b/pkg/migrate/attributemigration.go index 4233b3369..5ef0364b2 100644 --- a/pkg/migrate/attributemigration.go +++ b/pkg/migrate/attributemigration.go @@ -1,12 +1,10 @@ package migrate import ( - "encoding/json" "fmt" "regexp" "sync" - "github.com/Axway/agent-sdk/pkg/api" v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" mv1a "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/management/v1alpha1" defs "github.com/Axway/agent-sdk/pkg/apic/definitions" @@ -49,6 +47,7 @@ type client interface { UpdateResourceInstance(ri v1.Interface) (*v1.ResourceInstance, error) CreateOrUpdateResource(data v1.Interface) (*v1.ResourceInstance, error) CreateSubResource(rm v1.ResourceMeta, subs map[string]interface{}) error + DeleteResourceInstance(ri v1.Interface) error } type item struct { @@ -60,16 +59,17 @@ type migrateFunc func(ri *v1.ResourceInstance) error // AttributeMigration - used for migrating attributes to subresource type AttributeMigration struct { - client client - cfg config.CentralConfig + migration riMutex sync.Mutex } // NewAttributeMigration creates a new AttributeMigration func NewAttributeMigration(client client, cfg config.CentralConfig) *AttributeMigration { return &AttributeMigration{ - client: client, - cfg: cfg, + migration: migration{ + client: client, + cfg: cfg, + }, riMutex: sync.Mutex{}, } } @@ -259,34 +259,6 @@ func (m *AttributeMigration) migrate(resourceURL string, query map[string]string return nil } -// updateRI updates the resource, and the sub resource -func (m *AttributeMigration) updateRI(ri *v1.ResourceInstance) error { - _, err := m.client.UpdateResourceInstance(ri) - if err != nil { - return err - } - - return m.createSubResource(ri) -} - -func (m *AttributeMigration) getRI(url string) (*v1.ResourceInstance, error) { - response, err := m.client.ExecuteAPI(api.GET, url, nil, nil) - if err != nil { - return nil, fmt.Errorf("error while retrieving ResourceInstance: %s", err) - } - - resourceInstance := &v1.ResourceInstance{} - err = json.Unmarshal(response, &resourceInstance) - return resourceInstance, err -} - -func (m *AttributeMigration) createSubResource(ri *v1.ResourceInstance) error { - subResources := map[string]interface{}{ - defs.XAgentDetails: ri.SubResources[defs.XAgentDetails], - } - return m.client.CreateSubResource(ri.ResourceMeta, subResources) -} - func updateAttrs(ri *v1.ResourceInstance) item { details := util.GetAgentDetails(ri) if details == nil { @@ -334,7 +306,3 @@ func updateAttrs(ri *v1.ResourceInstance) item { return item } - -func queryFunc(name string) string { - return fmt.Sprintf("metadata.references.name==%s", name) -} diff --git a/pkg/migrate/attributemigration_test.go b/pkg/migrate/attributemigration_test.go index e96f77f7f..7e9a56f20 100644 --- a/pkg/migrate/attributemigration_test.go +++ b/pkg/migrate/attributemigration_test.go @@ -178,3 +178,7 @@ func (m *mockAttrMigClient) CreateOrUpdateResource(data apiv1.Interface) (*apiv1 func (m *mockAttrMigClient) ExecuteAPI(_, _ string, _ map[string]string, _ []byte) ([]byte, error) { return json.Marshal(m.execRes) } + +func (m mockAttrMigClient) DeleteResourceInstance(ri apiv1.Interface) error { + return nil +} diff --git a/pkg/migrate/marketplacemigration.go b/pkg/migrate/marketplacemigration.go index 102bb8f76..e4f0e0f55 100644 --- a/pkg/migrate/marketplacemigration.go +++ b/pkg/migrate/marketplacemigration.go @@ -13,7 +13,6 @@ import ( "github.com/Axway/agent-sdk/pkg/apic/definitions" "github.com/Axway/agent-sdk/pkg/apic/provisioning" "github.com/Axway/agent-sdk/pkg/config" - "github.com/Axway/agent-sdk/pkg/util" "github.com/Axway/agent-sdk/pkg/util/log" ) @@ -36,6 +35,13 @@ type ardCache interface { GetAccessRequestDefinitionByName(name string) (*v1.ResourceInstance, error) } +// MarketplaceMigration - used for migrating attributes to subresource +type MarketplaceMigration struct { + migration + logger log.FieldLogger + cache ardCache +} + // NewMarketplaceMigration - creates a new MarketplaceMigration func NewMarketplaceMigration(client client, cfg config.CentralConfig, cache ardCache) *MarketplaceMigration { logger := log.NewFieldLogger(). @@ -43,21 +49,15 @@ func NewMarketplaceMigration(client client, cfg config.CentralConfig, cache ardC WithComponent("MarketplaceMigration") return &MarketplaceMigration{ + migration: migration{ + client: client, + cfg: cfg, + }, logger: logger, - client: client, - cfg: cfg, cache: cache, } } -// MarketplaceMigration - used for migrating attributes to subresource -type MarketplaceMigration struct { - logger log.FieldLogger - client client - cfg config.CentralConfig - cache ardCache -} - // Migrate - func (m *MarketplaceMigration) Migrate(ri *v1.ResourceInstance) (*v1.ResourceInstance, error) { if ri.Kind != mv1a.APIServiceGVK().Kind { @@ -71,16 +71,12 @@ func (m *MarketplaceMigration) Migrate(ri *v1.ResourceInstance) (*v1.ResourceIns } // get x-agent-details and determine if we need to process this apiservice for marketplace provisioning - details := util.GetAgentDetails(apiSvc) - if len(details) > 0 { - completed := details[definitions.MarketplaceMigration] - if completed == definitions.MigrationCompleted { - // migration ran already - m.logger. - WithField(serviceName, apiSvc). - Debugf("marketplace provision migration already completed") - return ri, nil - } + if isMigrationCompleted(ri, definitions.MarketplaceMigration) { + // migration ran already + m.logger. + WithField(serviceName, apiSvc). + Debugf("marketplace provision migration already completed") + return ri, nil } m.logger. @@ -251,16 +247,6 @@ func (m *MarketplaceMigration) registerAccessRequestDefinition(scopes map[string return ard, nil } -// updateRI updates the resource, and the sub resource -func (m *MarketplaceMigration) updateRI(ri *v1.ResourceInstance) error { - _, err := m.client.UpdateResourceInstance(ri) - if err != nil { - return err - } - - return nil -} - func (m *MarketplaceMigration) handleSvcInstance( svcInstance *v1.ResourceInstance, revision *v1.ResourceInstance) error { logger := m.logger. @@ -280,7 +266,6 @@ func (m *MarketplaceMigration) handleSvcInstance( if processor, ok := i.(apic.OasSpecProcessor); ok { ardRIName := apiSvcInst.Spec.AccessRequestDefinition - credentialRequestPolicies := apiSvcInst.Spec.CredentialRequestDefinitions processor.ParseAuthInfo() @@ -310,7 +295,7 @@ func (m *MarketplaceMigration) handleSvcInstance( } // Check if CRD exists - credentialRequestPolicies, err = getCredentialRequestPolicies(authPolicies) + credentialRequestPolicies, err := getCredentialRequestPolicies(authPolicies) if err != nil { return err } diff --git a/pkg/migrate/marketplacemigration_test.go b/pkg/migrate/marketplacemigration_test.go index 00ee60d1e..f57723b04 100644 --- a/pkg/migrate/marketplacemigration_test.go +++ b/pkg/migrate/marketplacemigration_test.go @@ -5,7 +5,7 @@ import ( "sync" "testing" - cache2 "github.com/Axway/agent-sdk/pkg/agent/cache" + "github.com/Axway/agent-sdk/pkg/agent/cache" apiv1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" mv1a "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/management/v1alpha1" @@ -22,10 +22,10 @@ func TestMarketplaceMigrationForAPIKeyRevision(t *testing.T) { rev := newRevision(svcName, "api-key-revision") c := &mockMPMigClient{ - revisions: []*apiv1.ResourceInstance{ + revisions: []*v1.ResourceInstance{ rev, }, - instances: []*apiv1.ResourceInstance{ + instances: []*v1.ResourceInstance{ newInstance(rev.Name, "inst1", "", []string{"api-key"}), newInstance(rev.Name, "inst2", "", []string{"api-key"}), }, @@ -34,7 +34,7 @@ func TestMarketplaceMigrationForAPIKeyRevision(t *testing.T) { Environment: envName, APIServerVersion: "v1alpha1", } - cm := cache2.NewAgentCacheManager(cfg, false) + cm := cache.NewAgentCacheManager(cfg, false) ard := mv1a.NewAccessRequestDefinition("api-key", envName) ard.Metadata.ID = "123" @@ -56,10 +56,10 @@ func TestMarketplaceMigrationForOAuthRevision(t *testing.T) { rev := newOAuthRev(svcName, "oauth-revision") c := &mockMPMigClient{ - revisions: []*apiv1.ResourceInstance{ + revisions: []*v1.ResourceInstance{ rev, }, - instances: []*apiv1.ResourceInstance{ + instances: []*v1.ResourceInstance{ newInstance(rev.Name, "inst1", "", []string{provisioning.OAuthPublicKeyCRD, provisioning.OAuthSecretCRD}), }, } @@ -67,7 +67,7 @@ func TestMarketplaceMigrationForOAuthRevision(t *testing.T) { Environment: envName, APIServerVersion: "v1alpha1", } - cm := cache2.NewAgentCacheManager(cfg, false) + cm := cache.NewAgentCacheManager(cfg, false) ard := mv1a.NewAccessRequestDefinition("oauth", envName) ard.Metadata.ID = "0012" @@ -87,7 +87,7 @@ func TestMarketplaceMigrationForOAuthRevision(t *testing.T) { assert.NotNil(t, a) } -func newInstance(revName, instName, ard string, credentialRequestDefinitions []string) *apiv1.ResourceInstance { +func newInstance(revName, instName, ard string, credentialRequestDefinitions []string) *v1.ResourceInstance { inst := mv1a.NewAPIServiceInstance(instName, envName) inst.Spec = mv1a.ApiServiceInstanceSpec{ ApiServiceRevision: revName, @@ -108,7 +108,7 @@ func newInstance(revName, instName, ard string, credentialRequestDefinitions []s return ri } -func newRevision(svcName, revName string) *apiv1.ResourceInstance { +func newRevision(svcName, revName string) *v1.ResourceInstance { rev := mv1a.NewAPIServiceRevision(revName, envName) rev.Spec = mv1a.ApiServiceRevisionSpec{ ApiService: svcName, @@ -140,15 +140,15 @@ type mockMPMigClient struct { sync.Mutex updateCount int createSubCalled bool - revisions []*apiv1.ResourceInstance - instances []*apiv1.ResourceInstance + revisions []*v1.ResourceInstance + instances []*v1.ResourceInstance } func (m *mockMPMigClient) ExecuteAPI(_, _ string, _ map[string]string, _ []byte) ([]byte, error) { return nil, nil } -func (m *mockMPMigClient) GetAPIV1ResourceInstancesWithPageSize(_ map[string]string, url string, _ int) ([]*apiv1.ResourceInstance, error) { +func (m *mockMPMigClient) GetAPIV1ResourceInstancesWithPageSize(_ map[string]string, url string, _ int) ([]*v1.ResourceInstance, error) { if strings.Contains(url, "instances") { return m.instances, nil } @@ -156,28 +156,34 @@ func (m *mockMPMigClient) GetAPIV1ResourceInstancesWithPageSize(_ map[string]str } -func (m *mockMPMigClient) UpdateAPIV1ResourceInstance(_ string, _ *apiv1.ResourceInstance) (*apiv1.ResourceInstance, error) { +func (m *mockMPMigClient) Updatev1ResourceInstance(_ string, _ *v1.ResourceInstance) (*v1.ResourceInstance, error) { return nil, nil } -func (m *mockMPMigClient) CreateSubResourceScoped(_ apiv1.ResourceMeta, _ map[string]interface{}) error { +func (m *mockMPMigClient) CreateSubResourceScoped(_ v1.ResourceMeta, _ map[string]interface{}) error { return nil } -func (m *mockMPMigClient) CreateSubResource(_ apiv1.ResourceMeta, _ map[string]interface{}) error { +func (m *mockMPMigClient) CreateSubResource(_ v1.ResourceMeta, _ map[string]interface{}) error { + m.Lock() + defer m.Unlock() m.createSubCalled = true return nil } -func (m *mockMPMigClient) UpdateResourceInstance(_ apiv1.Interface) (*apiv1.ResourceInstance, error) { +func (m *mockMPMigClient) UpdateResourceInstance(_ v1.Interface) (*v1.ResourceInstance, error) { m.Lock() defer m.Unlock() m.updateCount = m.updateCount + 1 return nil, nil } -func (m *mockMPMigClient) CreateOrUpdateResource(i apiv1.Interface) (*apiv1.ResourceInstance, error) { +func (m *mockMPMigClient) CreateOrUpdateResource(i v1.Interface) (*v1.ResourceInstance, error) { ri, _ := i.AsInstance() ri.Metadata.ID = ardID return ri, nil } + +func (m *mockMPMigClient) DeleteResourceInstance(ri apiv1.Interface) error { + return nil +} diff --git a/pkg/migrate/migration.go b/pkg/migrate/migration.go new file mode 100644 index 000000000..6d8c4b31f --- /dev/null +++ b/pkg/migrate/migration.go @@ -0,0 +1,70 @@ +package migrate + +import ( + "encoding/json" + "fmt" + + "github.com/Axway/agent-sdk/pkg/api" + v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1" + "github.com/Axway/agent-sdk/pkg/apic/definitions" + defs "github.com/Axway/agent-sdk/pkg/apic/definitions" + "github.com/Axway/agent-sdk/pkg/config" + "github.com/Axway/agent-sdk/pkg/util" +) + +// migration - used for migrating resources +type migration struct { + client client + cfg config.CentralConfig +} + +// updateRI updates the resource, and the sub resource +func (m *migration) updateRI(ri *v1.ResourceInstance) error { + _, err := m.client.UpdateResourceInstance(ri) + if err != nil { + return err + } + + return m.createSubResource(ri) +} + +func (m *migration) createSubResource(ri *v1.ResourceInstance) error { + subResources := map[string]interface{}{ + defs.XAgentDetails: ri.SubResources[defs.XAgentDetails], + } + return m.client.CreateSubResource(ri.ResourceMeta, subResources) +} + +func (m *migration) getRI(url string) (*v1.ResourceInstance, error) { + response, err := m.client.ExecuteAPI(api.GET, url, nil, nil) + if err != nil { + return nil, fmt.Errorf("error while retrieving ResourceInstance: %s", err) + } + + resourceInstance := &v1.ResourceInstance{} + err = json.Unmarshal(response, &resourceInstance) + return resourceInstance, err +} + +func (m *migration) getAllRI(url string, q map[string]string) ([]*v1.ResourceInstance, error) { + resources, err := m.client.GetAPIV1ResourceInstancesWithPageSize(q, url, 100) + if err != nil { + return nil, fmt.Errorf("error while retrieving all ResourceInstances: %s", err) + } + return resources, nil +} + +func queryFunc(name string) string { + return fmt.Sprintf("metadata.references.name==%s", name) +} + +func isMigrationCompleted(h v1.Interface, migrationKey string) bool { + details := util.GetAgentDetails(h) + if len(details) > 0 { + completed := details[migrationKey] + if completed == definitions.MigrationCompleted { + return true + } + } + return false +}