From 8ddfdc26bb843b4c51592a8258954c317f60be83 Mon Sep 17 00:00:00 2001 From: yingzhan Date: Fri, 25 Oct 2024 14:32:09 +0800 Subject: [PATCH] OCM-10805 | test: automated ids:76394,76395 --- tests/ci/data/profiles/rosa-hcp.yaml | 4 + tests/e2e/hcp_cluster_test.go | 223 ++++++++++++++++++ tests/utils/config/cluster.go | 7 + tests/utils/exec/rosacli/cluster_service.go | 40 ++-- tests/utils/helper/certificate.go | 52 ++++ tests/utils/profilehandler/interface.go | 3 + tests/utils/profilehandler/profile_handler.go | 29 +++ 7 files changed, 340 insertions(+), 18 deletions(-) create mode 100644 tests/utils/helper/certificate.go diff --git a/tests/ci/data/profiles/rosa-hcp.yaml b/tests/ci/data/profiles/rosa-hcp.yaml index 98cd2a8ada..e1d51405d2 100644 --- a/tests/ci/data/profiles/rosa-hcp.yaml +++ b/tests/ci/data/profiles/rosa-hcp.yaml @@ -29,6 +29,8 @@ profiles: disable_uwm: true autoscaler_enabled: false additional_sg_number: 3 + registries_config: true + allowed_registries: true account-role: customized_prefix: true path: "" @@ -55,6 +57,8 @@ profiles: label_enabled: false tag_enabled: false zones: "" + registries_config: true + blocked_registries: true account-role: path: "/test/hcp/" permission_boundary: "arn:aws:iam::aws:policy/AdministratorAccess" diff --git a/tests/e2e/hcp_cluster_test.go b/tests/e2e/hcp_cluster_test.go index 196ab7eeae..0de2006ee8 100644 --- a/tests/e2e/hcp_cluster_test.go +++ b/tests/e2e/hcp_cluster_test.go @@ -2,6 +2,7 @@ package e2e import ( "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -434,4 +435,226 @@ var _ = Describe("HCP cluster testing", Expect(err).To(BeNil()) Expect(CD.AWSBillingAccount).To(Equal(constants.BillingAccount)) }) + + It("create ROSA HCP with registry config can work well via rosa cli - [id:76394]", + labels.High, labels.Runtime.Day1Post, + func() { + By("Check the help message of 'rosa create cluster -h'") + helpOutput, err, _ := clusterService.Create("", "-h") + Expect(err).To(BeNil()) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-allowed-registries")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-insecure-registries")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-blocked-registries")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-allowed-registries-for-import")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-additional-trusted-ca")) + + By("Check if cluster enable registry config") + output, err := clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err := clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + + if clusterConfig.RegistryConfig { + Skip("It is only for registry config enabled clusters") + } + jsonData, err := clusterService.GetJSONClusterDescription(clusterID) + Expect(err).To(BeNil()) + + for _, v := range clusterDetail.RegistryConfiguration { + if v["Allowed Registries"] != nil { + allowedList := jsonData.DigString("registry_config", "registry_sources", "allowed_registries") + if len(allowedList) > 2 { + result := strings.Replace(allowedList, " ", ",", 1) + Expect(result).To(Equal(fmt.Sprintf("[%s]", v["Allowed Registries"]))) + } + } + if v["Blocked Registries"] != nil { + blockedList := jsonData.DigString("registry_config", "registry_sources", "blocked_registries") + if len(blockedList) > 2 { + result := strings.Replace(blockedList, " ", ",", 1) + Expect(result).To(Equal(fmt.Sprintf("[%s]", v["Blocked Registries"]))) + } + } + if v["Insecure Registries"] != nil { + insecureList := jsonData.DigString("registry_config", "registry_sources", "insecure_registries") + if len(insecureList) > 2 { + result := strings.Replace(insecureList, " ", ",", 1) + Expect(result).To(Equal(fmt.Sprintf("[%s]", v["Insecure Registries"]))) + } + } + if v["Allowed Registries for Import"] != nil { + clusterData := jsonData.DigObject("registry_config", "allowed_registries_for_import") + if clusterData != "" { + allowedImport := v["Allowed Registries for Import"].([]interface{}) + for _, a := range clusterData.([]interface{}) { + importListFromJson := a.(map[string]interface{}) + insecureValue := false + if importListFromJson["insecure"] != nil { + insecureValue = importListFromJson["insecure"].(bool) + } + value1 := map[string]interface{}{ + "Domain Name": importListFromJson["domain_name"], + } + value2 := map[string]interface{}{ + "Insecure": insecureValue, + } + Expect(allowedImport).To(ContainElement(value1)) + Expect(allowedImport).To(ContainElement(value2)) + } + } + } + if v["Platform Allowlist"] != nil { + platformListID := jsonData.DigString("registry_config", "platform_allowlist", "id") + pList := v["Platform Allowlist"].([]interface{}) + for _, p := range pList { + pMap := p.(map[string]interface{}) + if pMap["ID"] != nil { + Expect(pMap["ID"].(string)).To(Equal(platformListID)) + } + } + } + + if v["Additional Trusted CA"] != nil { + caContent := jsonData.DigObject("registry_config", "additional_trusted_ca") + if caContent != "" { + caFromc := caContent.(map[string]interface{}) + for _, ca := range v["Additional Trusted CA"].([]interface{}) { + Expect(caFromc).To(Equal(ca)) + } + } + } + } + }) + + It("edit ROSA HCP with registry config can work well via rosa cli - [id:76395]", + labels.High, labels.Runtime.Day2, + func() { + By("Check the help message of 'rosa edit cluster -h'") + helpOutput, err := clusterService.EditCluster("", "-h") + Expect(err).To(BeNil()) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-allowed-registries")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-insecure-registries")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-blocked-registries")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-allowed-registries-for-import")) + Expect(helpOutput.String()).To(ContainSubstring("--registry-config-additional-trusted-ca")) + + By("Edit hcp cluster with registry configs") + if clusterConfig.RegistryConfig { + Skip("It is only for registry config enabled clusters") + } + + output, err := clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err := clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + for _, v := range clusterDetail.RegistryConfiguration { + if v["Allowed Registries"] != nil { + By("Remove allowed registry config") + originValue := v["Allowed Registries"].(string) + out, err := clusterService.EditCluster(clusterID, + "--registry-config-allowed-registries", "", + "-y", + ) + Expect(err).ToNot(HaveOccurred()) + textData := rosaClient.Parser.TextData.Input(out).Parse().Tip() + Expect(textData).To(ContainSubstring("Updated cluster '%s'", clusterID)) + + By("Describe cluster to check the value") + output, err := clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err = clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[0]["Allowed Registries"]).To(BeNil()) + + By("Add blocked registry config") + blockedValue := "test.blocked.com,*.example.com" + out, err = clusterService.EditCluster(clusterID, + "--registry-config-blocked-registries", blockedValue, + "-y", + ) + Expect(err).ToNot(HaveOccurred()) + textData = rosaClient.Parser.TextData.Input(out).Parse().Tip() + Expect(textData).To(ContainSubstring("Updated cluster '%s'", clusterID)) + + By("Describe cluster to check the value") + output, err = clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err = clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[1]["Blocked Registries"]).To(Equal(blockedValue)) + + By("Update it back") + out, err = clusterService.EditCluster(clusterID, + "--registry-config-blocked-registries", "", + "--registry-config-allowed-registries", originValue, + "-y", + ) + Expect(err).ToNot(HaveOccurred()) + textData = rosaClient.Parser.TextData.Input(out).Parse().Tip() + Expect(textData).To(ContainSubstring("Updated cluster '%s'", clusterID)) + + By("Describe cluster to check the value") + output, err = clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err = clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[0]["Allowed Registries"]).To(Equal(originValue)) + Expect(clusterDetail.RegistryConfiguration[1]["Blocked Registries"]).To(BeNil()) + + } + if v["Blocked Registries"] != nil { + By("Remove blocked registry config") + originValue := v["Blocked Registries"].(string) + out, err := clusterService.EditCluster(clusterID, + "--registry-config-blocked-registries", "", + "-y", + ) + Expect(err).ToNot(HaveOccurred()) + textData := rosaClient.Parser.TextData.Input(out).Parse().Tip() + Expect(textData).To(ContainSubstring("Updated cluster '%s'", clusterID)) + + By("Describe cluster to check the value") + output, err := clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err = clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[0]["Blocked Registries"]).To(BeNil()) + + By("Add allowed registry config") + allowedValue := "test.allowed.com,*.example.com" + out, err = clusterService.EditCluster(clusterID, + "--registry-config-allowed-registries", allowedValue, + "-y", + ) + Expect(err).ToNot(HaveOccurred()) + textData = rosaClient.Parser.TextData.Input(out).Parse().Tip() + Expect(textData).To(ContainSubstring("Updated cluster '%s'", clusterID)) + + By("Describe cluster to check the value") + output, err = clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err = clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[0]["Allowed Registries"]).To(Equal(allowedValue)) + + By("Update it back") + out, err = clusterService.EditCluster(clusterID, + "--registry-config-blocked-registries", originValue, + "--registry-config-allowed-registries", "", + "-y", + ) + Expect(err).ToNot(HaveOccurred()) + textData = rosaClient.Parser.TextData.Input(out).Parse().Tip() + Expect(textData).To(ContainSubstring("Updated cluster '%s'", clusterID)) + + By("Describe cluster to check the value") + output, err = clusterService.DescribeCluster(clusterID) + Expect(err).To(BeNil()) + clusterDetail, err = clusterService.ReflectClusterDescription(output) + Expect(err).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[0]["Allowed Registries"]).To(BeNil()) + Expect(clusterDetail.RegistryConfiguration[1]["Blocked Registries"]).To(Equal(originValue)) + } + } + }) }) diff --git a/tests/utils/config/cluster.go b/tests/utils/config/cluster.go index 31fb80499c..9788c1ebdb 100644 --- a/tests/utils/config/cluster.go +++ b/tests/utils/config/cluster.go @@ -99,6 +99,12 @@ type AdditionalSecurityGroups struct { InfraSecurityGroups string `json:"infra_sgs,omitempty"` WorkerSecurityGroups string `json:"worker_sgs,omitempty"` } +type RegistryConfig struct { + AllowedRegistries string `json:"allowed_registries,omitempty"` + RegistryAdditionalTrustCA string `json:"reristry_additional_trust_ca,omitempty"` + AllowedRegistriesForImport []string `json:"allowed_registries_for_import,omitempty"` +} + type ClusterConfig struct { DisableScpChecks bool `json:"disable_scp_checks,omitempty"` DisableWorkloadMonitoring bool `json:"disable_workload_monitoring,omitempty"` @@ -135,6 +141,7 @@ type ClusterConfig struct { Version *Version `json:"version,omitempty"` ExternalAuthentication bool `json:"external_authentication,omitempty"` SharedVPC bool `json:"shared_vpc,omitempty"` + RegistryConfig bool `json:"registry_config,omitempty"` } func ParseClusterProfile() (*ClusterConfig, error) { diff --git a/tests/utils/exec/rosacli/cluster_service.go b/tests/utils/exec/rosacli/cluster_service.go index 59d6dd57fd..b9f7152cc5 100644 --- a/tests/utils/exec/rosacli/cluster_service.go +++ b/tests/utils/exec/rosacli/cluster_service.go @@ -99,24 +99,26 @@ type ClusterDescription struct { Proxy []map[string]string `yaml:"Proxy,omitempty"` STSRoleArn string `yaml:"Role (STS) ARN,omitempty"` // STSExternalID string `yaml:"STS External ID,omitempty"` - SupportRoleARN string `yaml:"Support Role ARN,omitempty"` - OperatorIAMRoles []string `yaml:"Operator IAM Roles,omitempty"` - InstanceIAMRoles []map[string]string `yaml:"Instance IAM Roles,omitempty"` - ManagedPolicies string `yaml:"Managed Policies,omitempty"` - UserWorkloadMonitoring string `yaml:"User Workload Monitoring,omitempty"` - FIPSMod string `yaml:"FIPS mode,omitempty"` - OIDCEndpointURL string `yaml:"OIDC Endpoint URL,omitempty"` - PrivateHostedZone []map[string]string `yaml:"Private Hosted Zone,omitempty"` - AuditLogForwarding string `yaml:"Audit Log Forwarding,omitempty"` - ProvisioningErrorMessage string `yaml:"Provisioning Error Message,omitempty"` - ProvisioningErrorCode string `yaml:"Provisioning Error Code,omitempty"` - LimitedSupport []map[string]string `yaml:"Limited Support,omitempty"` - AuditLogRoleARN string `yaml:"Audit Log Role ARN,omitempty"` - FailedInflightChecks string `yaml:"Failed Inflight Checks,omitempty"` - ExternalAuthentication string `yaml:"External Authentication,omitempty"` - EnableDeleteProtection string `yaml:"Delete Protection,omitempty"` - EnableEtcdEncryption string `yaml:"Etcd Encryption,omitempty"` - EtcdKmsKeyARN string `yaml:"Etcd KMS key ARN,omitempty"` + SupportRoleARN string `yaml:"Support Role ARN,omitempty"` + OperatorIAMRoles []string `yaml:"Operator IAM Roles,omitempty"` + InstanceIAMRoles []map[string]string `yaml:"Instance IAM Roles,omitempty"` + ManagedPolicies string `yaml:"Managed Policies,omitempty"` + UserWorkloadMonitoring string `yaml:"User Workload Monitoring,omitempty"` + FIPSMod string `yaml:"FIPS mode,omitempty"` + OIDCEndpointURL string `yaml:"OIDC Endpoint URL,omitempty"` + PrivateHostedZone []map[string]string `yaml:"Private Hosted Zone,omitempty"` + AuditLogForwarding string `yaml:"Audit Log Forwarding,omitempty"` + ProvisioningErrorMessage string `yaml:"Provisioning Error Message,omitempty"` + ProvisioningErrorCode string `yaml:"Provisioning Error Code,omitempty"` + LimitedSupport []map[string]string `yaml:"Limited Support,omitempty"` + AuditLogRoleARN string `yaml:"Audit Log Role ARN,omitempty"` + FailedInflightChecks string `yaml:"Failed Inflight Checks,omitempty"` + ExternalAuthentication string `yaml:"External Authentication,omitempty"` + EnableDeleteProtection string `yaml:"Delete Protection,omitempty"` + EnableEtcdEncryption string `yaml:"Etcd Encryption,omitempty"` + EtcdKmsKeyARN string `yaml:"Etcd KMS key ARN,omitempty"` + RegistryConfiguration []map[string]interface{} `yaml:"Registry Configuration,omitempty"` + ZeroEgress string `yaml:"Zero Egress,omitempty"` } // Pasrse the result of 'rosa list cluster' to the ClusterList struct @@ -197,6 +199,8 @@ func (c *clusterService) ReflectClusterDescription(result bytes.Buffer) (res *Cl newStr = strings.Replace(str, "Failed Inflight Checks:", "Failed Inflight Checks: |", 1) newStr = strings.ReplaceAll(newStr, "\t", " ") newStr = strings.ReplaceAll(newStr, "not found: Role name", "not found:Role name") + //Until https://issues.redhat.com/browse/OCM-11830 fixed + newStr = strings.Replace(newStr, "Platform Allowlist:", "Platform Allowlist: \n - ID:", 1) return }). YamlToMap() diff --git a/tests/utils/helper/certificate.go b/tests/utils/helper/certificate.go new file mode 100644 index 0000000000..d20e9e49e8 --- /dev/null +++ b/tests/utils/helper/certificate.go @@ -0,0 +1,52 @@ +package helper + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" +) + +// Create a PEM Certificate +// Code taken from https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ +func CreatePEMCertificate() (string, error) { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Red Hat"}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"Raleigh"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return "", err + } + + caPEM := new(bytes.Buffer) + err = pem.Encode(caPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + if err != nil { + return "", err + } + return caPEM.String(), nil +} diff --git a/tests/utils/profilehandler/interface.go b/tests/utils/profilehandler/interface.go index 6484c185f1..fb9869c799 100644 --- a/tests/utils/profilehandler/interface.go +++ b/tests/utils/profilehandler/interface.go @@ -57,6 +57,9 @@ type ClusterConfig struct { SharedVPC bool `yaml:"shared_vpc,omitempty" json:"shared_vpc,omitempty"` TagEnabled bool `yaml:"tag_enabled,omitempty" json:"tag_enabled,omitempty"` NetworkType string `yaml:"network_type,omitempty" json:"network_type,omitempty"` + RegistriesConfig bool `yaml:"registries_config" json:"registries_config,omitempty"` + AllowedRegistries bool `yaml:"allowed_registries" json:"allowed_registries,omitempty"` + BlockedRegistries bool `yaml:"blocked_registries" json:"blocked_registries,omitempty"` } // UserData will record the user data prepared for resource clean up diff --git a/tests/utils/profilehandler/profile_handler.go b/tests/utils/profilehandler/profile_handler.go index 8b8ec61e59..ea5fdedb1a 100644 --- a/tests/utils/profilehandler/profile_handler.go +++ b/tests/utils/profilehandler/profile_handler.go @@ -681,6 +681,35 @@ func GenerateClusterCreateFlags(profile *Profile, client *rosacli.Client) ([]str clusterConfiguration.Networking.Type = profile.ClusterConfig.NetworkType } + if profile.ClusterConfig.RegistriesConfig && profile.ClusterConfig.HCP { + caContent, err := helper.CreatePEMCertificate() + + if err != nil { + return flags, err + } + registryConfigCa := map[string]string{ + "test.io": caContent, + } + caFile := fmt.Sprintf("%s-%s", config.Test.OutputDir, "registryConfig") + _, err = helper.CreateFileWithContent(caFile, registryConfigCa) + if err != nil { + return flags, err + } + flags = append(flags, + "--registry-config-additional-trusted-ca", caFile, + "--registry-config-insecure-registries", "test.com,*.example", + "--registry-config-allowed-registries-for-import", "example.com:true,test.com:false", + ) + if profile.ClusterConfig.AllowedRegistries { + flags = append(flags, + "--registry-config-allowed-registries", "allowed.example.com,*.test", + ) + } else { + flags = append(flags, + "--registry-config-blocked-registries", "blocked.example.com,*.test", + ) + } + } return flags, nil } func WaitForClusterPassWaiting(client *rosacli.Client, cluster string, timeoutMin int) error {