*: add conversion logic (#764)

* *: add conversion logic

* *: helm support for v3.0.0 apis

* *: v1beta1  backup api

* backup*: rewrite backup v1beta1 api

* chart: fix  imageNamespaceOverride bug

*: add nfs backup support

* *: add nfs backup support

* *: add copyCopyright header and move mysqlchecker to init-sidecar

* test*: fix error when running tests

---------

Co-authored-by: aceking <[email protected]>

.dockerignore

@@ -6,4 +6,5 @@
 .git
 bin
 testbin
-vendor
+vendor
+test

Dockerfile

@@ -24,7 +24,7 @@ COPY utils/ utils/
 COPY mysqluser/ mysqluser/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -oPg manager cmd/manager/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager cmd/manager/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Dockerfile.sidecar

@@ -23,7 +23,11 @@ COPY sidecar/ sidecar/
 COPY utils/ utils/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o bin/sidecar cmd/sidecar/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o bin/sidecar cmd/sidecar/main.go
+
+# Build mysql checker for mysql conatiner
+COPY cmd/mysql/main.go cmd/mysql/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o bin/mysqlchecker cmd/mysql/main.go
 
 ###############################################################################
 #  Docker image for Sidecar
@@ -52,4 +56,5 @@ RUN set -ex; \
 
 WORKDIR /
 COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
+COPY --from=builder /workspace/bin/mysqlchecker /mnt/mysqlchecker
 ENTRYPOINT ["sidecar"]

Makefile

@@ -1,17 +1,14 @@
 # Image URL to use all building/pushing image targets
-FROM_VERSION ?=v2.2.1
-CHART_VERSION ?=2.2.1
-CHART_TOVERSION ?=2.3.0
-TO_VERSION ?=v2.3.0
+FROM_VERSION ?=v2.3.0
+CHART_VERSION ?=2.3.0
+CHART_TOVERSION ?=3.0.0
+TO_VERSION ?=v3.0.0
+TAG ?=v3.0.0
 IMGPREFIX ?=radondb/
-MYSQL_IMAGE_57 ?=5.7.39
-MYSQL_IMAGE_80 ?=8.0.26
-MYSQL_IMAGE_57_TAG ?=$(IMGPREFIX)percona-server:$(MYSQL_IMAGE_57)
-MYSQL_IMAGE_80_TAG ?=$(IMGPREFIX)percona-server:$(MYSQL_IMAGE_80)
-IMG ?= $(IMGPREFIX)mysql-operator:latest
-SIDECAR57_IMG ?= $(IMGPREFIX)mysql57-sidecar:latest
-SIDECAR80_IMG ?= $(IMGPREFIX)mysql80-sidecar:latest
-XENON_IMG ?= $(IMGPREFIX)xenon:latest
+IMG ?= $(IMGPREFIX)mysql-operator:$(TAG)
+SIDECAR57_IMG ?= $(IMGPREFIX)mysql57-sidecar:$(TAG)
+SIDECAR80_IMG ?= $(IMGPREFIX)mysql80-sidecar:$(TAG)
+XENON_IMG ?= $(IMGPREFIX)xenon:$(TAG)
 GO_PORXY ?= off
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
@@ -56,15 +53,24 @@ update-crd:	## Synchronize the generated YAML files to operator Chart after make
 	make manifests
 	cp config/crd/bases/* charts/mysql-operator/crds/
 
-generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate: controller-gen generate-go-conversions ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 fmt: ## Run go fmt against code.
 	go fmt ./...
 
 vet: ## Run go vet against code.
 	go vet ./...
-
+
+CONVERSION_GEN := $(shell pwd)/bin/conversion-gen
+CODE_GENERATOR_VERSION := $(shell awk '/k8s.io\/client-go/ {print substr($$2, 2)}' go.mod)
+conversion-gen: ## Donwload conversion-gen locally if necessary.
+	$(call go-get-tool,$(CONVERSION_GEN),k8s.io/code-generator/cmd/conversion-gen@v$(CODE_GENERATOR_VERSION))
+generate-go-conversions: conversion-gen $(CONVERSION_GEN) ## Generate conversions go code
+	$(CONVERSION_GEN) \
+		--input-dirs=./api/v1beta1 \
+		--output-file-base=zz_generated.conversion --output-base=. \
+		--go-header-file=./hack/boilerplate.go.txt
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test: manifests generate fmt vet ## Run tests.
 	mkdir -p ${ENVTEST_ASSETS_DIR}
@@ -81,15 +87,11 @@ build: generate fmt vet ## Build manager binary.
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/manager/main.go
 
-docker-build: test ## Build docker image with the manager.
+docker-build:  ## Build docker image with the manager.
 	docker buildx build --build-arg GO_PROXY=${GO_PORXY} -t ${IMG} .
 	docker buildx build -f Dockerfile.sidecar --build-arg GO_PROXY=${GO_PORXY} -t ${SIDECAR57_IMG} .
 	docker buildx build -f build/xenon/Dockerfile --build-arg GO_PROXY=${GO_PORXY} -t ${XENON_IMG} .
 	docker buildx build  --build-arg XTRABACKUP_PKG=percona-xtrabackup-80  --build-arg GO_PROXY=${GO_PORXY} -f  Dockerfile.sidecar -t ${SIDECAR80_IMG} .
-	docker buildx build  --build-arg "MYSQL_IMAGE=${MYSQL_IMAGE_57}"  --build-arg GO_PROXY=${GO_PORXY} -f build/mysql/Dockerfile  -t ${MYSQL_IMAGE_57_TAG}   .
-	docker buildx build  --build-arg "MYSQL_IMAGE=${MYSQL_IMAGE_80}"  --build-arg GO_PROXY=${GO_PORXY} -f build/mysql/Dockerfile  -t ${MYSQL_IMAGE_80_TAG}   .
-docker-build-mysql57: test ## Build docker image with the manager.
-	docker buildx build  --build-arg "MYSQL_IMAGE=${MYSQL_IMAGE_57}"  --build-arg GO_PROXY=${GO_PORXY} -f build/mysql/Dockerfile  -t ${MYSQL_IMAGE_57_TAG}   .
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
 	docker push ${SIDECAR_IMG}
@@ -141,13 +143,20 @@ todo:
 	@grep -Irnw './' -e 'TODO:'|grep -v grep
 
 updateVersion:
-	find ./ -type f -name "*.go" -o -name "*.yaml" -exec sed -i "s/mysql57-sidecar:$(FROM_VERSION)/mysql57-sidecar:$(TO_VERSION)/g" {} \;
-	find ./ -type f -name "*.go" -o -name "*.yaml" -exec sed -i "s/xenon:$(FROM_VERSION)/xenon:$(TO_VERSION)/g" {} \;
-	find ./ -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "s/mysql-operator:$(FROM_VERSION)/mysql-operator:$(TO_VERSION)/g" {} \;
-	find ./ -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "s/mysql80-sidecar:$(FROM_VERSION)/mysql80-sidecar:$(TO_VERSION)/g" {} \;
-	find ./ -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "s/mysql-operator-$(FROM_VERSION)/mysql-operator-$(TO_VERSION)/g" {} \;
-	find ./ -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "s/\"$(FROM_VERSION)\"/\"$(TO_VERSION)\"/g" {} \;
+	find ./* -type f -name "*.go" -o -name "*.yaml" -exec sed -i "" "s/mysql57-sidecar:$(FROM_VERSION)/mysql57-sidecar:$(TO_VERSION)/g" {} \;
+	find ./* -type f -name "*.go" -o -name "*.yaml" -exec sed -i "" "s/xenon:$(FROM_VERSION)/xenon:$(TO_VERSION)/g" {} \;
+	find ./* -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "" "s/mysql-operator:$(FROM_VERSION)/mysql-operator:$(TO_VERSION)/g" {} \;
+	find ./* -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "" "s/mysql80-sidecar:$(FROM_VERSION)/mysql80-sidecar:$(TO_VERSION)/g" {} \;
+	find ./* -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "" "s/mysql-operator-$(FROM_VERSION)/mysql-operator-$(TO_VERSION)/g" {} \;
+	find ./* -type f -name "*.go" -o -name "*.yaml" -exec sed -i  "" "s/\"$(FROM_VERSION)\"/\"$(TO_VERSION)\"/g" {} \;
 	# sed -i "18s/$(CHART_VERSION)/$(CHART_TOVERSION)/"  charts/mysql-operator/charts/Chart.yaml
-	find ./charts  -type f -name "*.yaml" -exec sed -i  "s/$(CHART_VERSION)/$(CHART_TOVERSION)/g" {} \;
-	find ./config  -type f -name "*.yaml" -exec sed -i  "s/$(CHART_VERSION)/$(CHART_TOVERSION)/g" {} \;
+	find ./charts/*  -type f -name "*.yaml" -exec sed -i  "" "s/$(CHART_VERSION)/$(CHART_TOVERSION)/g" {} \;
+	find ./config/*  -type f -name "*.yaml" -exec sed -i  "" "s/$(CHART_VERSION)/$(CHART_TOVERSION)/g" {} \;
 
+CRD_TO_MARKDOWN := $(shell pwd)/bin/crd-to-markdown
+CRD_TO_MARKDOWN_VERSION = 0.0.3
+crd-to-markdown: ## Download crd-to-markdown locally if necessary.
+	$(call go-get-tool,$(CRD_TO_MARKDOWN),github.com/clamoriniere/crd-to-markdown@v$(CRD_TO_MARKDOWN_VERSION))
+apidoc: crd-to-markdown $(wildcard api/*/*_types.go)
+	$(CRD_TO_MARKDOWN) --links docs/links.csv -f api/v1beta1/mysqlcluster_types.go  -n MySQLCluster > docs/crd_mysqlcluster_v1beta1.md
+

api/v1alpha1/doc.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1

api/v1alpha1/mysqlcluster_conversion.go

@@ -1,3 +1,20 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package v1alpha1
 
 func (*MysqlCluster) Hub() {}
+func (*Backup) Hub()       {}

api/v1beta1/backup_types.go

@@ -28,56 +28,158 @@ import (
 type BackupSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
+	// ClusterName is the name of the cluster to be backed up.
+	ClusterName string `json:"clusterName,omitempty"`
+	// BackupMethod represents the type of backup
+	BackupMethod string `json:"method,omitempty"`
+	// Defines details for manual  backup Jobs
+	// +optional
+	Manual *ManualBackup `json:"manual,omitempty"`
+	// Backup Schedule
+	// +optional
+	BackupSchedule *BackupSchedule `json:"schedule,omitempty"`
+	// Backup Storage
+	BackupOpts BackupOps `json:"backupops,omitempty"`
+}
 
-	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
+type BackupOps struct {
+	// BackupHost
+	// +optional
+	BackupHost string `json:"host,omitempty"`
+	S3         *S3    `json:"s3,omitempty"`
+	NFS        *NFS   `json:"nfs,omitempty"`
+}
 
-	// To specify the image that will be used for sidecar container.
+type S3 struct {
+	// S3 Bucket
 	// +optional
-	// +kubebuilder:default:="radondb/mysql57-sidecar:v2.3.0"
-	Image string `json:"image"`
+	BackupSecretName string `json:"secretName,omitempty"`
+}
 
-	// HostFrom represents the host for which to take backup
-	// If is empty, is use leader HostFrom
-	HostFrom string `json:"hostfrom,omitempty"`
+type NFS struct {
+	// Defines a Volume for backup MySQL data.
+	// More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes
+	Volume corev1.NFSVolumeSource `json:"volume,omitempty"`
+}
 
-	// Represents the ip address of the nfs server.
+type ManualBackup struct {
+	BackupType string `json:"type,omitempty"`
+	// Backup Retention
 	// +optional
-	NFSServerAddress string `json:"nfsServerAddress,omitempty"`
-
-	// ClusterName represents the cluster name to backup
-	ClusterName string `json:"clusterName"`
+	// +kubebuilder:default:=7
+	BackupRetention *int32 `json:"backupRetention,omitempty"`
+}
 
+type BackupSchedule struct {
+	// Cron expression for backup schedule
+	// +optional
+	CronExpression string `json:"cronExpression,omitempty"`
+	// Backup Retention
+	// +optional
+	BackupRetention *int32 `json:"backupRetention,omitempty"`
+	BackupType      string `json:"type,omitempty"`
 	// History Limit of job
 	// +optional
-	// +kubebuilder:default:=3
-	HistoryLimit *int32 `json:"historyLimit,omitempty"`
+	BackupJobHistoryLimit *int32 `json:"jobhistoryLimit,omitempty"`
 }
 
-// BackupStatus defines the observed state of Backup
 type BackupStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
-	// +kubebuilder:default:=false
-	Completed bool `json:"completed"`
-	// Get the backup path.
+	Type             BackupInitiator         `json:"type,omitempty"`
+	BackupName       string                  `json:"backupName,omitempty"`
+	BackupSize       string                  `json:"backupSize,omitempty"`
+	BackupType       string                  `json:"backupType,omitempty"`
+	StartTime        *metav1.Time            `json:"startTime,omitempty"`
+	CompletionTime   *metav1.Time            `json:"completionTime,omitempty"`
+	State            BackupConditionType     `json:"state,omitempty"`
+	ManualBackup     *ManualBackupStatus     `json:"manual,omitempty"`
+	ScheduledBackups []ScheduledBackupStatus `json:"scheduled,omitempty"`
+}
+
+type BackupConditionType string
+
+const (
+	// BackupComplete means the backup has finished his execution
+	BackupSucceeded BackupConditionType = "Succeeded"
+	// BackupFailed means backup has failed
+	BackupFailed BackupConditionType = "Failed"
+	BackupStart  BackupConditionType = "Started"
+	BackupActive BackupConditionType = "Active"
+)
+
+type BackupInitiator string
+
+const (
+	CronJobBackupInitiator BackupInitiator = "CronJob"
+	ManualBackupInitiator  BackupInitiator = "Manual"
+)
+
+type ManualBackupStatus struct {
+	// Specifies whether or not the Job is finished executing (does not indicate success or
+	// failure).
+	// +kubebuilder:validation:Required
+	Finished   bool   `json:"finished"`
 	BackupName string `json:"backupName,omitempty"`
 	// Get the backup Date
-	BackupDate string `json:"backupDate,omitempty"`
+	StartTime *metav1.Time `json:"startTime,omitempty"`
+	// Get the backup Type
+	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
+	// Conditions represents the backup resource conditions list.
+	// +optional
+	Succeeded int32 `json:"succeeded,omitempty"`
+	// +optional
+	// The number of actively running manual backup Pods.
+	// +optional
+	Active int32  `json:"active,omitempty"`
+	Failed int32  `json:"failed,omitempty"`
+	Reason string `json:"reason"`
 	// Get the backup Type
 	BackupType string `json:"backupType,omitempty"`
+	// Get the backup Size
+	BackupSize string `json:"backupSize,omitempty"`
+	// Get current backup status
+	State BackupConditionType `json:"state,omitempty"`
+}
+
+type ScheduledBackupStatus struct {
+	// The name of the associated  scheduled backup CronJob
+	// +kubebuilder:validation:Required
+	CronJobName string `json:"cronJobName,omitempty"`
+	// Get the backup path.
+	BackupName string `json:"backupName,omitempty"`
+	// Specifies whether or not the Job is finished executing (does not indicate success or
+	// failure).
+	// +kubebuilder:validation:Required
+	Finished bool `json:"finished"`
+	// Get the backup Type
+	BackupType string `json:"backupType,omitempty"`
+	// Get the backup Date
+	StartTime *metav1.Time `json:"startTime,omitempty"`
+	// Get the backup Type
+	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
 	// Conditions represents the backup resource conditions list.
-	Conditions []BackupCondition `json:"conditions,omitempty"`
+	// +optional
+	Succeeded int32 `json:"succeeded,omitempty"`
+	// +optional
+	Failed int32  `json:"failed,omitempty"`
+	Reason string `json:"reason"`
+	// Get the backup Size
+	BackupSize string `json:"backupSize,omitempty"`
+	// Get current backup status
+	State BackupConditionType `json:"state,omitempty"`
 }
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:storageversion
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="BackupName",type="string",JSONPath=".status.backupName",description="The Backup name"
-// +kubebuilder:printcolumn:name="BackupDate",type="string",JSONPath=".status.backupDate",description="The Backup Date time"
+// +kubebuilder:printcolumn:name="StartTime",type="string",JSONPath=".status.startTime",description="The Backup Start time"
+// +kubebuilder:printcolumn:name="completionTime",type="string",JSONPath=".status.completionTime",description="The Backup CompletionTime time"
 // +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.backupType",description="The Backup Type"
-// +kubebuilder:printcolumn:name="Success",type="string",JSONPath=".status.conditions[?(@.type==\"Complete\")].status",description="Whether the backup Success?"
+// +kubebuilder:printcolumn:name="Initiator",type="string",JSONPath=".status.type",description="The Backup Initiator"
+// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.state",description="The Backup State"
+// +kubebuilder:printcolumn:name="Size",type="string",JSONPath=".status.backupSize",description="The Backup State"
+
 // Backup is the Schema for the backups API
 type Backup struct {
 	metav1.TypeMeta   `json:",inline"`
@@ -87,31 +189,6 @@ type Backup struct {
 	Status BackupStatus `json:"status,omitempty"`
 }
 
-// BackupCondition defines condition struct for backup resource
-type BackupCondition struct {
-	// type of cluster condition, values in (\"Ready\")
-	Type BackupConditionType `json:"type"`
-	// Status of the condition, one of (\"True\", \"False\", \"Unknown\")
-	Status corev1.ConditionStatus `json:"status"`
-	// LastTransitionTime
-	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
-	// Reason
-	Reason string `json:"reason"`
-	// Message
-	Message string `json:"message"`
-}
-
-// BackupConditionType defines condition types of a backup resources
-type BackupConditionType string
-
-const (
-	// BackupComplete means the backup has finished his execution
-	BackupComplete BackupConditionType = "Complete"
-	// BackupFailed means backup has failed
-	BackupFailed BackupConditionType = "Failed"
-	BackupStart  BackupConditionType = "Started"
-)
-
 //+kubebuilder:object:root=true
 
 // BackupList contains a list of Backup