Skip to content

Commit

Permalink
sidecar: remove xtrabackup copy back #691 (#692)
Browse files Browse the repository at this point in the history
  • Loading branch information
@acekingke
acekingke authored Sep 30, 2022
1 parent adeeb4a commit be4421e
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 51 deletions.
59 changes: 24 additions & 35 deletions sidecar/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,10 +519,6 @@ func (cfg *Config) executeS3Restore(path string) error {
return fmt.Errorf("failed to create data directory : %s", err)
}
}
// mkdir /root/backup.
if err := os.MkdirAll("/root/backup", 0755); err != nil {
return fmt.Errorf("failed to create backup directory : %s", err)
}
// Execute xbcloud get.
args := []string{
"get",
Expand All @@ -535,8 +531,8 @@ func (cfg *Config) executeS3Restore(path string) error {
cfg.XRestoreFrom,
"--insecure",
}
xcloud := exec.Command(xcloudCommand, args...) //nolint
xbstream := exec.Command("xbstream", "-xv", "-C", "/root/backup") //nolint
xcloud := exec.Command(xcloudCommand, args...) //nolint
xbstream := exec.Command("xbstream", "-xv", "-C", utils.DataVolumeMountPath) //nolint
var err error
if xbstream.Stdin, err = xcloud.StdoutPipe(); err != nil {
return fmt.Errorf("failed to xbstream and xcloud piped")
Expand Down Expand Up @@ -564,31 +560,25 @@ func (cfg *Config) executeS3Restore(path string) error {
}
}
// Xtrabackup prepare and apply-log-only.
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--prepare", "--apply-log-only", "--target-dir=/root/backup")
log.Info("Xtrabackup prepare and apply-log-only")
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--prepare", "--apply-log-only", "--target-dir="+utils.DataVolumeMountPath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare and apply-log-only : %s", err)
}
// Xtrabackup prepare.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--prepare", "--target-dir=/root/backup")
log.Info("Xtrabackup prepare")
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--prepare", "--target-dir="+utils.DataVolumeMountPath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare : %s", err)
}
// Xtrabackup copy-back to /var/lib/mysql.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--datadir="+utils.DataVolumeMountPath, "--copy-back", "--copy-back", "--target-dir=/root/backup")
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup copy-back : %s", err)
}
// Do not need to Xtrabackup copy-back to /var/lib/mysql.
// Execute chown -R mysql.mysql /var/lib/mysql.
log.Info("chown -R mysql.mysql /var/lib/mysql")
if err := exec.Command("chown", "-R", "mysql.mysql", utils.DataVolumeMountPath).Run(); err != nil {
return fmt.Errorf("failed to chown mysql.mysql %s : %s", utils.DataVolumeMountPath, err)
}
// Remove /root/backup.
if err := os.RemoveAll("/root/backup"); err != nil {
return fmt.Errorf("failed to remove backup directory : %s", err)
}
return nil
}

Expand All @@ -597,40 +587,38 @@ func (cfg *Config) executeCloneRestore() error {
// Check directory exist, create if not exist.
if _, err := os.Stat(utils.DataVolumeMountPath); os.IsNotExist(err) {
os.Mkdir(utils.DataVolumeMountPath, 0755)
// Empty the directory. just for lost+found.
dir, err := ioutil.ReadDir(utils.DataVolumeMountPath)
if err != nil {
return fmt.Errorf("failed to read datadir %s", err)
}
for _, d := range dir {
os.RemoveAll(path.Join([]string{utils.DataVolumeMountPath, d.Name()}...))
}
}

// Empty the directory.
dir, err := ioutil.ReadDir(utils.DataVolumeMountPath)
if err != nil {
return fmt.Errorf("failed to read datadir %s", err)
}
for _, d := range dir {
os.RemoveAll(path.Join([]string{utils.DataVolumeMountPath, d.Name()}...))
}
// Xtrabackup prepare and apply-log-only.
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--apply-log-only", "--target-dir=/backup/"+cfg.XRestoreFrom)
log.Info("xtrabackup prepare apply-log only")
cmd := exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--apply-log-only", "--target-dir="+utils.DataVolumeMountPath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare apply-log-only : %s", err)
}
// Xtrabackup Prepare.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--target-dir=/backup/"+cfg.XRestoreFrom)
log.Info("xtrabackup prepare")
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--target-dir="+utils.DataVolumeMountPath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup prepare : %s", err)
}
// Get the backup binlong info.
gtid, err := GetXtrabackupGTIDPurged("/backup/" + cfg.XRestoreFrom)
gtid, err := GetXtrabackupGTIDPurged(utils.DataVolumeMountPath)
if err == nil {
cfg.GtidPurged = gtid
}
log.Info("get master gtid purged :", "gtid purged", cfg.GtidPurged)
// Xtrabackup copy-back.
cmd = exec.Command(xtrabackupCommand, "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--datadir="+utils.DataVolumeMountPath, "--copy-back", "--target-dir=/backup/"+cfg.XRestoreFrom)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to xtrabackup copy-back : %s", err)
}
// Do not need Xtrabackup copy-back.

// Remove Relaybin.
// Because the relaybin is not used in the restore process,
// we can remove it to prevent it to be used by salve in the future.
Expand All @@ -640,6 +628,7 @@ func (cfg *Config) executeCloneRestore() error {
return fmt.Errorf("failed to remove relay-bin : %s", err)
}
// Run chown -R mysql.mysql /var/lib/mysql
log.Info("Run chown -R mysql.mysql /var/lib/mysql")
cmd = exec.Command("chown", "-R", "mysql.mysql", utils.DataVolumeMountPath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
Expand Down
29 changes: 16 additions & 13 deletions sidecar/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,12 @@ func NewInitCommand(cfg *Config) *cobra.Command {
Use: "init",
Short: "do some initialization operations.",
Run: func(cmd *cobra.Command, args []string) {
if err := runCloneAndInit(cfg); err != nil {
var init bool
var err error
if init, err = runCloneAndInit(cfg); err != nil {
log.Error(err, "clone error")
}
if err := runInitCommand(cfg); err != nil {
if err = runInitCommand(cfg, init); err != nil {
log.Error(err, "init command failed")
os.Exit(1)
}
Expand Down Expand Up @@ -85,9 +87,10 @@ func CheckServiceExist(cfg *Config, service string) bool {
}

// Clone from leader or follower.
func runCloneAndInit(cfg *Config) error {
func runCloneAndInit(cfg *Config) (bool, error) {
//check follower is exists?
serviceURL := ""
var hasInitialized = false
// Check the rebuildFrom exist?
if service, err := getPod(cfg); err == nil {
serviceURL = service
Expand All @@ -105,30 +108,30 @@ func runCloneAndInit(cfg *Config) error {

if len(serviceURL) != 0 {
// Check has initialized. If so just return.
hasInitialized, _ := checkIfPathExists(path.Join(dataPath, "mysql"))
hasInitialized, _ = checkIfPathExists(path.Join(dataPath, "mysql"))
if hasInitialized {
log.Info("MySQL data directory existing!")
return nil
return hasInitialized, nil
}
// backup at first
Args := fmt.Sprintf("rm -rf /backup/initbackup;mkdir -p /backup/initbackup;curl --user $BACKUP_USER:$BACKUP_PASSWORD %s/download|xbstream -x -C /backup/initbackup; exit ${PIPESTATUS[0]}",
serviceURL)
Args := fmt.Sprintf("curl --user $BACKUP_USER:$BACKUP_PASSWORD %s/download|xbstream -x -C %s; exit ${PIPESTATUS[0]}",
serviceURL, utils.DataVolumeMountPath)
cmd := exec.Command("/bin/bash", "-c", "--", Args)
log.Info("runCloneAndInit", "cmd", Args)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to disable the run restore: %s", err)
return hasInitialized, fmt.Errorf("failed to disable the run restore: %s", err)
}
cfg.XRestoreFrom = backupInitDirectory
cfg.XRestoreFrom = utils.DataVolumeMountPath // just for init clone
cfg.CloneFlag = true
return nil
return hasInitialized, nil
}
log.Info("no leader or follower found")
return nil
return hasInitialized, nil
}

// runInitCommand do some initialization operations.
func runInitCommand(cfg *Config) error {
func runInitCommand(cfg *Config, hasInitialized bool) error {
var err error
// Get the mysql user.
user, err := user.Lookup("mysql")
Expand Down Expand Up @@ -190,7 +193,7 @@ func runInitCommand(cfg *Config) error {
if err = ioutil.WriteFile(initFilePath+"/reset.sql", []byte("reset master;"), 0644); err != nil {
return fmt.Errorf("failed to write reset.sql: %s", err)
}
hasInitialized, _ := checkIfPathExists(path.Join(dataPath, "mysql"))

// build init.sql.
initSqlPath := path.Join(extraConfPath, "init.sql")

Expand Down
3 changes: 0 additions & 3 deletions sidecar/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,6 @@ var (

// xcloudCommand is the upload tool file name.
xcloudCommand = "xbcloud"

// clone restore init data directory.
backupInitDirectory = "initbackup"
)

// copyFile the src file to dst.
Expand Down

0 comments on commit be4421e

Please sign in to comment.