Skip to content

Commit

Permalink
Merge branch 'main' into fix-monitoring
Browse files Browse the repository at this point in the history
  • Loading branch information
tplavcic authored Dec 16, 2024
2 parents bf056cd + a223c04 commit 72f0c29
Show file tree
Hide file tree
Showing 530 changed files with 17,317 additions and 8,739 deletions.
2 changes: 1 addition & 1 deletion .github/linters/go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
module linters

go 1.13
go 1.23.4
1 change: 1 addition & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

**Config/Logging/Testability**
- [ ] Are all needed new/changed options added to default YAML files?
- [ ] Are all needed new/changed options added to the [Helm Chart](https://github.com/percona/percona-helm-charts)?
- [ ] Did we add proper logging messages for operator actions?
- [ ] Did we ensure compatibility with the previous version or cluster upgrade process?
- [ ] Does the change support oldest and newest supported PXC version?
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/reviewdog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.21'
go-version: '1.23'
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
Expand All @@ -34,7 +34,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.21'
go-version: '1.23'
- run: go install mvdan.cc/sh/v3/cmd/shfmt@latest
- run: $(go env GOPATH)/bin/shfmt -f . | grep -v 'vendor' | xargs $(go env GOPATH)/bin/shfmt -bn -ci -s -w
- name: suggester / shfmt
Expand Down Expand Up @@ -85,7 +85,7 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.21'
go-version: '1.23'
- name: check on release branch
if: ${{ contains(github.head_ref, 'release-') || contains(github.base_ref, 'release-') }}
run: |
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,15 @@ jobs:
./e2e-tests/build
- name: Run Trivy vulnerability scanner image (linux/arm64)
uses: aquasecurity/trivy-action@0.23.0
uses: aquasecurity/trivy-action@0.29.0
with:
image-ref: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-arm64'
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
version: 'v0.57.1'

- name: Build an image from Dockerfile (linux/amd64)
run: |
Expand All @@ -49,11 +50,12 @@ jobs:
./e2e-tests/build
- name: Run Trivy vulnerability scanner image (linux/amd64)
uses: aquasecurity/trivy-action@0.23.0
uses: aquasecurity/trivy-action@0.29.0
with:
image-ref: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-amd64'
format: 'table'
exit-code: '1'
ignore-unfixed: true
vuln-type: 'os,library'
severity: 'CRITICAL,HIGH'
version: 'v0.57.1'
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
steps:
- uses: actions/setup-go@v5
with:
go-version: '1.21'
go-version: '1.23'
- uses: actions/checkout@v4
- name: go test
run: make test
6 changes: 3 additions & 3 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ void createCluster(String CLUSTER_SUFFIX) {
gcloud config set project $GCP_PROJECT
gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.26 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \
gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \
kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
if [ \${ret_val} -eq 0 ]; then break; fi
ret_num=\$((ret_num + 1))
Expand Down Expand Up @@ -361,7 +361,7 @@ EOF
--rm \
-v $WORKSPACE/src/github.com/percona/percona-xtradb-cluster-operator:/go/src/github.com/percona/percona-xtradb-cluster-operator \
-w /go/src/github.com/percona/percona-xtradb-cluster-operator \
golang:1.22 sh -c '
golang:1.23 sh -c '
go install -mod=readonly github.com/google/go-licenses@latest;
/go/bin/go-licenses csv github.com/percona/percona-xtradb-cluster-operator/cmd/manager \
| cut -d , -f 3 \
Expand Down Expand Up @@ -390,7 +390,7 @@ EOF
-w /go/src/github.com/percona/percona-xtradb-cluster-operator \
-e GO111MODULE=on \
-e GOFLAGS='-buildvcs=false' \
golang:1.22 sh -c 'go build -v -o percona-xtradb-cluster-operator github.com/percona/percona-xtradb-cluster-operator/cmd/manager'
golang:1.23 sh -c 'go build -v -o percona-xtradb-cluster-operator github.com/percona/percona-xtradb-cluster-operator/cmd/manager'
"
'''

Expand Down
22 changes: 21 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ vet: ## Run go vet against code.
test: manifests generate fmt vet envtest ## Run tests.
DISABLE_TELEMETRY=true KUBEBUILDER_ASSETS="$(shell $(ENVTEST) --arch=amd64 use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out

e2e-test:
IMAGE=$(IMAGE) ./e2e-tests/$(TEST)/run

##@ Build

.PHONY: build
Expand Down Expand Up @@ -88,7 +91,7 @@ endef

CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
controller-gen: ## Download controller-gen locally if necessary.
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0)
$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.3)

KUSTOMIZE = $(shell pwd)/bin/kustomize
kustomize: ## Download kustomize locally if necessary.
Expand All @@ -98,6 +101,10 @@ ENVTEST = $(shell pwd)/bin/setup-envtest
envtest: ## Download envtest-setup locally if necessary.
$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)

SWAGGER = $(shell pwd)/bin/swagger
swagger: ## Download swagger locally if necessary.
$(call go-get-tool,$(SWAGGER),github.com/go-swagger/go-swagger/cmd/swagger@latest)

# Prepare release
CERT_MANAGER_VER := $(shell grep -Eo "cert-manager v.*" go.mod|grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
release: manifests
Expand Down Expand Up @@ -138,3 +145,16 @@ after-release: manifests
-e "/^ backup:/,/^ image:/{s#image: .*#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#}" \
-e "/initContainer:/,/image:/{s#image: .*#image: perconalab/percona-xtradb-cluster-operator:main#}" \
-e "/^ pmm:/,/^ image:/{s#image: .*#image: perconalab/pmm-client:dev-latest#}" deploy/cr.yaml

VS_BRANCH = main
version-service-client: swagger
curl https://raw.githubusercontent.com/Percona-Lab/percona-version-service/$(VS_BRANCH)/api/version.swagger.yaml \
--output ./version.swagger.yaml
rm -rf ./version/client
mkdir -p ./version/client/models
mkdir -p ./version/client/version_service
./bin/swagger generate client \
-f ./version.swagger.yaml \
-c ./version/client \
-m ./version/client/models
rm ./version.swagger.yaml
40 changes: 11 additions & 29 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ Within the [Percona Operator for MySQL based on Percona XtraDB Cluster](https://
* Automated Password Rotation – use the standard Kubernetes API to enforce password rotation policies for system user
* Private container image registries

You interact with Percona Operator mostly via the command line tool. If you feel more comfortable with operating the Operator and database clusters via the web interface, there is [Percona Everest](https://docs.percona.com/everest/index.html) - an open-source web-based database provisioning tool available for you. It automates day-to-day database management operations for you, reducing the overall administrative overhead. [Get started with Percona Everest](https://docs.percona.com/everest/quickstart-guide/quick-install.html).

While the Percona Operator is primarily managed through the command line, you can also use **[Percona Everest](https://docs.percona.com/everest/index.html)** for a web-based user interface. This open-source tool provides a streamlined experience for provisioning and managing your databases, simplifying day-to-day tasks and reducing administrative overhead. Learn more about Percona Everest in the [documentation](https://docs.percona.com/everest/index.html) or jump right in with the [quickstart guide](https://docs.percona.com/everest/quickstart-guide/quick-install.html).

# Architecture

Expand Down Expand Up @@ -57,41 +56,24 @@ kubectl apply -f https://raw.githubusercontent.com/percona/percona-xtradb-cluste

See full documentation with examples and various advanced cases on [percona.com](https://www.percona.com/doc/kubernetes-operator-for-pxc/index.html).

# Need help?

**Commercial Support** | **Community Support** |
:-: | :-: |
| <br/>Enterprise-grade assistance for your mission-critical MySQL deployments with the Percona Operator for MySQL. Get expert guidance for complex tasks like multi-cloud replication, database migration and building platforms.<br/><br/> | <br/>Connect with our engineers and fellow users for general questions, troubleshooting, and sharing feedback and ideas.<br/><br/> |
| **[Get Percona Support](https://hubs.ly/Q02ZTH940)** | **[Visit our Forum](https://forums.percona.com/c/mysql-mariadb/percona-kubernetes-operator-for-mysql/28)** |

# Contributing

Percona welcomes and encourages community contributions to help improve Percona Operator for MySQL.

See the [Contribution Guide](CONTRIBUTING.md) and [Building and Testing Guide](e2e-tests/README.md) for more information on how you can contribute.

## Communication

We would love to hear from you! Reach out to us on [Forum](https://forums.percona.com/c/mysql-mariadb/percona-kubernetes-operator-for-mysql/28) with your questions, feedback and ideas

# Join Percona Kubernetes Squad!
```
% _____
%%% | __ \
###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _
### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` |
#### ##% %%%% | | | __/ | | (_| (_) | | | | (_| |
### #### %%% |_| \___|_| \___\___/|_| |_|\__,_|
,((### ### %%% _ _ _____ _
(((( (### #### %%%% | | / _ \ / ____| | |
((( ((# ###### | | _| (_) |___ | (___ __ _ _ _ __ _ __| |
(((( (((# #### | |/ /> _ </ __| \___ \ / _` | | | |/ _` |/ _` |
/(( ,((( *### | <| (_) \__ \ ____) | (_| | |_| | (_| | (_| |
//// ((( #### |_|\_\\___/|___/ |_____/ \__, |\__,_|\__,_|\__,_|
/// (((( #### | |
/////////////(((((((((((((((((######## |_| Join @ percona.com/k8s
```

You can get early access to new product features, invite-only ”ask me anything” sessions with Percona Kubernetes experts, and monthly swag raffles. Interested? Fill in the form at [percona.com/k8s](https://www.percona.com/k8s).

# Roadmap
## Roadmap

We have an experimental public roadmap which can be found [here](https://github.com/percona/roadmap/projects/1). Please feel free to contribute and propose new features by following the roadmap [guidelines](https://github.com/percona/roadmap).
We have a public roadmap which can be found [here](https://github.com/orgs/percona/projects/10). Please feel free to contribute and propose new features by following the roadmap [guidelines](https://github.com/percona/roadmap).

# Submitting Bug Reports
## Submitting Bug Reports

If you find a bug in Percona Docker Images or in one of the related projects, please submit a report to that project's [JIRA](https://jira.percona.com/browse/K8SPXC) issue tracker or [create a GitHub issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-an-issue#creating-an-issue-from-a-repository) in this repository.

Expand Down
25 changes: 24 additions & 1 deletion build/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM --platform=${BUILDPLATFORM} golang:1.22 AS go_builder
FROM --platform=${BUILDPLATFORM} golang:1.23 AS go_builder
WORKDIR /go/src/github.com/percona/percona-xtradb-cluster-operator

COPY go.mod go.sum ./
Expand Down Expand Up @@ -30,6 +30,11 @@ RUN GOOS=$GOOS GOARCH=${TARGETARCH} CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFL
cmd/pitr/main.go \
&& cp -r build/_output/bin/pitr /usr/local/bin/pitr

RUN GOOS=$GOOS GOARCH=${TARGETARCH} CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAGS \
go build -ldflags "-w -s -X main.GitCommit=$GIT_COMMIT -X main.GitBranch=$GIT_BRANCH -X main.BuildTime=$BUILD_TIME" \
-o build/_output/bin/mysql-state-monitor cmd/mysql-state-monitor/main.go \
&& cp -r build/_output/bin/mysql-state-monitor /usr/local/bin/mysql-state-monitor

# Looking for all possible License/Notice files and copying them to the image
RUN find $GOPATH/pkg/mod -regextype posix-extended -iregex '.*(license|notice)(\.md|\.txt|$)' \
-exec \
Expand All @@ -49,6 +54,7 @@ COPY LICENSE /licenses/
COPY --from=go_builder /usr/local/bin/percona-xtradb-cluster-operator /usr/local/bin/percona-xtradb-cluster-operator
COPY --from=go_builder /usr/local/bin/peer-list /peer-list
COPY --from=go_builder /usr/local/bin/pitr /pitr
COPY --from=go_builder /usr/local/bin/mysql-state-monitor /mysql-state-monitor
COPY build/pxc-entrypoint.sh /pxc-entrypoint.sh
COPY build/pxc-init-entrypoint.sh /pxc-init-entrypoint.sh
COPY build/pitr-init-entrypoint.sh /pitr-init-entrypoint.sh
Expand All @@ -60,4 +66,21 @@ COPY build/readiness-check.sh /readiness-check.sh
COPY build/pmm-prerun.sh /pmm-prerun.sh
COPY build/get-pxc-state /get-pxc-state

COPY build/haproxy-entrypoint.sh /haproxy-entrypoint.sh
COPY build/haproxy-init-entrypoint.sh /haproxy-init-entrypoint.sh
COPY build/haproxy_add_pxc_nodes.sh /haproxy_add_pxc_nodes.sh
COPY build/haproxy_check_pxc.sh /haproxy_check_pxc.sh
COPY build/haproxy_liveness_check.sh /haproxy_liveness_check.sh
COPY build/haproxy_readiness_check.sh /haproxy_readiness_check.sh
COPY build/haproxy.cfg /haproxy.cfg
COPY build/haproxy-global.cfg /haproxy-global.cfg

COPY build/proxysql-entrypoint.sh /proxysql-entrypoint.sh
COPY build/proxysql-init-entrypoint.sh /proxysql-init-entrypoint.sh
COPY build/proxysql.cnf /proxysql.cnf
COPY build/proxysql-admin.cnf /proxysql-admin.cnf
COPY build/proxysql_add_cluster_nodes.sh /proxysql_add_cluster_nodes.sh
COPY build/proxysql_add_proxysql_nodes.sh /proxysql_add_proxysql_nodes.sh
COPY build/proxysql_add_pxc_nodes.sh /proxysql_add_pxc_nodes.sh

USER 2
40 changes: 40 additions & 0 deletions build/haproxy-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/sh
set -e

log() {
local message="$1"
local date=$(/usr/bin/date +"%d/%b/%Y:%H:%M:%S.%3N")

echo "{\"time\":\"${date}\", \"message\": \"${message}\"}"
}

if [ "$1" = 'haproxy' ]; then
if [ ! -f '/etc/haproxy/pxc/haproxy.cfg' ]; then
log /opt/percona/haproxy.cfg /etc/haproxy/pxc
cp /opt/percona/haproxy.cfg /etc/haproxy/pxc
fi

custom_conf='/etc/haproxy-custom/haproxy-global.cfg'
if [ -f "$custom_conf" ]; then
log "haproxy -c -f $custom_conf -f /etc/haproxy/pxc/haproxy.cfg"
haproxy -c -f $custom_conf -f /etc/haproxy/pxc/haproxy.cfg || EC=$?
if [ -n "$EC" ]; then
log "The custom config $custom_conf is not valid and will be ignored."
fi
fi

haproxy_opt='-W -db '
if [ -f "$custom_conf" -a -z "$EC" ]; then
haproxy_opt+="-f $custom_conf "
else
haproxy_opt+='-f /opt/percona/haproxy-global.cfg '
fi

haproxy_opt+='-f /etc/haproxy/pxc/haproxy.cfg -p /etc/haproxy/pxc/haproxy.pid -S /etc/haproxy/pxc/haproxy-main.sock '
fi

log 'test -e /opt/percona/hookscript/hook.sh && source /opt/percona/hookscript/hook.sh'
test -e /opt/percona/hookscript/hook.sh && source /opt/percona/hookscript/hook.sh

log "$@ $haproxy_opt"
exec "$@" $haproxy_opt
51 changes: 51 additions & 0 deletions build/haproxy-global.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
global
log stdout format raw local0
maxconn 2048
external-check
insecure-fork-wanted
hard-stop-after 10s
stats socket /etc/haproxy/pxc/haproxy.sock mode 600 expose-fd listeners level admin

defaults
no option dontlognull
log-format '{"time":"%t", "client_ip": "%ci", "client_port":"%cp", "backend_source_ip": "%bi", "backend_source_port": "%bp", "frontend_name": "%ft", "backend_name": "%b", "server_name":"%s", "tw": "%Tw", "tc": "%Tc", "Tt": "%Tt", "bytes_read": "%B", "termination_state": "%ts", "actconn": "%ac", "feconn" :"%fc", "beconn": "%bc", "srv_conn": "%sc", "retries": "%rc", "srv_queue": "%sq", "backend_queue": "%bq" }'
default-server init-addr last,libc,none
log global
mode tcp
retries 10
timeout client 28800s
timeout connect 100500
timeout server 28800s

resolvers kubernetes
parse-resolv-conf

frontend galera-in
bind *:3309 accept-proxy
bind *:3306
mode tcp
option clitcpka
default_backend galera-nodes

frontend galera-admin-in
bind *:33062
mode tcp
option clitcpka
default_backend galera-admin-nodes

frontend galera-replica-in
bind *:3307
mode tcp
option clitcpka
default_backend galera-replica-nodes

frontend galera-mysqlx-in
bind *:33060
mode tcp
option clitcpka
default_backend galera-mysqlx-nodes

frontend stats
bind *:8404
mode http
http-request use-service prometheus-exporter if { path /metrics }
14 changes: 14 additions & 0 deletions build/haproxy-init-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/bin/bash

set -o errexit
set -o xtrace

install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy-entrypoint.sh /opt/percona/haproxy-entrypoint.sh
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy_check_pxc.sh /opt/percona/haproxy_check_pxc.sh
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy_add_pxc_nodes.sh /opt/percona/haproxy_add_pxc_nodes.sh
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy_readiness_check.sh /opt/percona/haproxy_readiness_check.sh
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy_liveness_check.sh /opt/percona/haproxy_liveness_check.sh
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy.cfg /opt/percona/haproxy.cfg
install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /haproxy-global.cfg /opt/percona/haproxy-global.cfg

install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /peer-list /opt/percona/peer-list
27 changes: 27 additions & 0 deletions build/haproxy.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
backend galera-nodes
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/haproxy_check_pxc.sh

backend galera-admin-nodes
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/haproxy_check_pxc.sh

backend galera-replica-nodes
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/haproxy_check_pxc.sh

backend galera-mysqlx-nodes
mode tcp
option srvtcpka
balance roundrobin
option external-check
external-check command /opt/percona/haproxy_check_pxc.sh
Loading

0 comments on commit 72f0c29

Please sign in to comment.