Skip to content

Commit

Permalink
[tlse] TLS database connection
Browse files Browse the repository at this point in the history 
The my.cnf file gets added to the secret holding the service configs.
The content of my.cnf is centrally managed in the mariadb-operator
and retrieved calling db.GetDatabaseClientConfig(tlsCfg)

Depends-On: openstack-k8s-operators/mariadb-operator#190
Depends-On: openstack-k8s-operators/mariadb-operator#191
Depends-On: openstack-k8s-operators/mariadb-operator#200

Jira: OSPRH-4547
  • Loading branch information
@stuggi
stuggi committed Feb 22, 2024
1 parent 69c5c53 commit fe5621a
Show file tree
Hide file tree
Showing 10 changed files with 324 additions and 70 deletions.
153 changes: 91 additions & 62 deletions controllers/glance_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"time"

"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
rbacv1 "k8s.io/api/rbac/v1"
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/fields"
Expand Down Expand Up @@ -363,67 +364,10 @@ func (r *GlanceReconciler) reconcileInit(
) (ctrl.Result, error) {
r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name))

//
// create service DB instance
//
db := mariadbv1.NewDatabase(
instance.Name,
instance.Spec.DatabaseUser,
instance.Spec.Secret,
map[string]string{
"dbName": instance.Spec.DatabaseInstance,
},
)
// create or patch the DB
ctrlResult, err := db.CreateOrPatchDB(
ctx,
helper,
)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return ctrl.Result{}, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return ctrlResult, nil
}
// wait for the DB to be setup
ctrlResult, err = db.WaitForDBCreated(ctx, helper)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return ctrlResult, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return ctrlResult, nil
}
// update Status.DatabaseHostname, used to config the service
instance.Status.DatabaseHostname = db.GetDatabaseHostname()
instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
// create service DB - end

//
// create Keystone service and users - https://docs.openstack.org/Glance/latest/install/install-rdo.html#configure-user-and-endpoints
//
_, _, err = oko_secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace)
_, _, err := oko_secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace)
if err != nil {
if k8s_errors.IsNotFound(err) {
return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Secret)
Expand All @@ -442,7 +386,7 @@ func (r *GlanceReconciler) reconcileInit(
}

ksSvc := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, time.Duration(10)*time.Second)
ctrlResult, err = ksSvc.CreateOrPatch(ctx, helper)
ctrlResult, err := ksSvc.CreateOrPatch(ctx, helper)
if err != nil {
return ctrlResult, err
}
Expand Down Expand Up @@ -588,12 +532,19 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
// run check OpenStack secret - end

db, result, err := r.ensureDB(ctx, helper, instance)
if err != nil {
return ctrl.Result{}, err
} else if (result != ctrl.Result{}) {
return result, nil
}

//
// Create Secrets required as input for the Service and calculate an overall hash of hashes
//

//
err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels)
err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels, db)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.ServiceConfigReadyCondition,
Expand Down Expand Up @@ -887,6 +838,7 @@ func (r *GlanceReconciler) generateServiceConfig(
instance *glancev1.Glance,
envVars *map[string]env.Setter,
serviceLabels map[string]string,
db *mariadbv1.Database,
) error {
labels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), serviceLabels)

Expand All @@ -899,7 +851,7 @@ func (r *GlanceReconciler) generateServiceConfig(
// hence only passing the database related parameters
templateParameters := map[string]interface{}{
"MinimalConfig": true, // This tells the template to generate a minimal config
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
instance.Spec.DatabaseUser,
string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
instance.Status.DatabaseHostname,
Expand All @@ -919,7 +871,17 @@ func (r *GlanceReconciler) generateServiceConfig(
templateParameters["ImageCacheDir"] = glance.ImageCacheDir
}

customData := map[string]string{glance.CustomConfigFileName: instance.Spec.CustomServiceConfig}
var tlsCfg *tls.Service
for _, api := range instance.Spec.GlanceAPIs {
if api.TLS.CaBundleSecretName != "" {
tlsCfg = &tls.Service{}
break
}
}
customData := map[string]string{
glance.CustomConfigFileName: instance.Spec.CustomServiceConfig,
"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
}

// Generate both default 00-config.conf and -scripts
return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, true)
Expand Down Expand Up @@ -1074,3 +1036,70 @@ func (r *GlanceReconciler) glanceAPICleanup(ctx context.Context, instance *glanc
}
return nil
}

func (r *GlanceReconciler) ensureDB(
ctx context.Context,
h *helper.Helper,
instance *glancev1.Glance,
) (*mariadbv1.Database, ctrl.Result, error) {
//
// create service DB instance
//
db := mariadbv1.NewDatabase(
instance.Name,
instance.Spec.DatabaseUser,
instance.Spec.Secret,
map[string]string{
"dbName": instance.Spec.DatabaseInstance,
},
)

// create or patch the DB
ctrlResult, err := db.CreateOrPatchDBByName(
ctx,
h,
instance.Spec.DatabaseInstance,
)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return db, ctrl.Result{}, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return db, ctrlResult, nil
}
// wait for the DB to be setup
// (ksambor) should we use WaitForDBCreatedWithTimeout instead?
ctrlResult, err = db.WaitForDBCreated(ctx, h)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return db, ctrlResult, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return db, ctrlResult, nil
}

// update Status.DatabaseHostname, used to config the service
instance.Status.DatabaseHostname = db.GetDatabaseHostname()
instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
return db, ctrlResult, nil
}
17 changes: 15 additions & 2 deletions controllers/glanceapi_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import (
"github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
"github.com/openstack-k8s-operators/lib-common/modules/common/util"
mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -791,8 +792,20 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
) error {
labels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), serviceLabels)

db, err := mariadbv1.GetDatabaseByName(ctx, h, glance.DatabaseName)
if err != nil {
return err
}

var tlsCfg *tls.Service
if instance.Spec.TLS.Ca.CaBundleSecretName != "" {
tlsCfg = &tls.Service{}
}
// 02-config.conf
customData := map[string]string{glance.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
customData := map[string]string{
glance.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig,
"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
}

// 03-config.conf
customSecrets := ""
Expand Down Expand Up @@ -845,7 +858,7 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
"ServicePassword": string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]),
"KeystoneInternalURL": keystoneInternalURL,
"KeystonePublicURL": keystonePublicURL,
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
instance.Spec.DatabaseUser,
string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
instance.Spec.DatabaseHostname,
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240216173409-86913e6d5885
github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240216173409-86913e6d5885
github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885
github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240215091212-cbf2ad281f43
github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e
k8s.io/api v0.28.3
k8s.io/apimachinery v0.28.3
k8s.io/client-go v0.28.3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.202402161
github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:sK82mkh2UzITsbNa/y6AKTZftHQnsYigqRx+rFbfZM4=
github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885 h1:ioJ2MO3vAcBkLM+0UBu5IuKW/DPXcyiNSOLq0Xvn+Nw=
github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:82nzS+DbBe1tzaMvNHH8FctmZzQ14ZAJysFGsMJiivo=
github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240215091212-cbf2ad281f43 h1:azblrnuVV8sLWihuqS7lJMrwpo1dtB1K5vvkug0agw4=
github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240215091212-cbf2ad281f43/go.mod h1:52Ja/B4RrrytMmKh+Kf+/BPe7Fq40Pi77vcFH4yJeoU=
github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e h1:6vqp5HZwcGvPH0MII/23iCd97T3/1HJZlONKW6LyNio=
github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down
15 changes: 15 additions & 0 deletions pkg/glance/cronjob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,28 @@ func CronJob(
},
},
},
{
Name: "config-data",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
DefaultMode: &config0644AccessMode,
SecretName: ServiceName + "-config-data",
},
},
},
}
cronJobVolumeMounts := []corev1.VolumeMount{
{
Name: "db-purge-config-data",
MountPath: "/etc/glance/glance.conf.d",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
}

// add CA cert if defined from the first api
Expand Down
6 changes: 6 additions & 0 deletions pkg/glance/dbsync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ func DbSyncJob(
MountPath: "/etc/glance/glance.conf.d",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/var/lib/kolla/config_files/config.json",
Expand Down
6 changes: 6 additions & 0 deletions pkg/glance/volumes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,12 @@ func GetVolumeMounts(secretNames []string, hasCinder bool, extraVol []glancev1.G
MountPath: "/var/lib/config-data/default",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
{
Name: ServiceName,
MountPath: "/var/lib/glance",
Expand Down
7 changes: 7 additions & 0 deletions test/functional/glance_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ var _ = Describe("Glance controller", func() {
th.SimulateJobSuccess(glanceTest.GlanceDBSync)
Glance := GetGlance(glanceTest.Instance)
Expect(Glance.Status.DatabaseHostname).To(Equal(fmt.Sprintf("hostname-for-openstack.%s.svc", namespace)))

secretDataMap := th.GetSecret(glanceTest.GlanceConfigMapData)
Expect(secretDataMap).ShouldNot(BeNil())
myCnf := secretDataMap.Data["my.cnf"]
Expect(myCnf).To(
ContainSubstring("[client]\nssl=0"))

th.ExpectCondition(
glanceName,
ConditionGetterFunc(GlanceConditionGetter),
Expand Down
Loading

0 comments on commit fe5621a

Please sign in to comment.