cmd: replacing release-20.2 version of roachprod, roachtest, workload

Release note: None

pkg/cmd/roachprod-stress/main.go

@@ -138,6 +138,16 @@ func run() error {
 		return err
 	}
 
+	const localLibDir = "lib.docker_amd64/"
+	if fi, err := os.Stat(localLibDir); err == nil && fi.IsDir() {
+		cmd = exec.Command("roachprod", "put", cluster, localLibDir, "lib")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return err
+		}
+	}
+
 	cmd = exec.Command("roachprod", "run", cluster, "mkdir -p "+pkg)
 	if err := cmd.Run(); err != nil {
 		return err

pkg/cmd/roachprod/install/cassandra.go

@@ -49,7 +49,7 @@ func (Cassandra) Start(c *SyncedCluster, extraArgs []string) {
 				_ = session.Close()
 			}()
 
-			cmd := c.Env + `env ROACHPROD=true cassandra` +
+			cmd := `env ` + c.Env + ` ROACHPROD=true cassandra` +
 				` -Dcassandra.config=file://${PWD}/cassandra.yaml` +
 				` -Dcassandra.ring_delay_ms=3000` +
 				` > cassandra.stdout 2> cassandra.stderr`

pkg/cmd/roachprod/install/cockroach.go

@@ -240,7 +240,7 @@ func (Cockroach) CertsDir(c *SyncedCluster, index int) string {
 func (Cockroach) NodeURL(c *SyncedCluster, host string, port int) string {
 	url := fmt.Sprintf("'postgres://root@%s:%d", host, port)
 	if c.Secure {
-		url += "?sslcert=certs%2Fnode.crt&sslkey=certs%2Fnode.key&" +
+		url += "?sslcert=certs%2Fclient.root.crt&sslkey=certs%2Fclient.root.key&" +
 			"sslrootcert=certs%2Fca.crt&sslmode=verify-full"
 	} else {
 		url += "?sslmode=disable"

pkg/cmd/roachprod/k8s/roachprod-gc.yaml

@@ -34,6 +34,7 @@ spec:
               image: gcr.io/cockroach-dev-inf/cockroachlabs/roachprod:master
               args:
                 - gc
+                -- --gce-project=cockroach-ephemeral,andrei-jepsen
                 - --slack-token
                 - $(SLACK_TOKEN)
               env:

pkg/cmd/roachprod/main.go

@@ -80,7 +80,7 @@ var (
 	listMine          bool
 	clusterType       = "cockroach"
 	secure            = false
-	nodeEnv           = "COCKROACH_ENABLE_RPC_COMPRESSION=false"
+	nodeEnv           = []string{"COCKROACH_ENABLE_RPC_COMPRESSION=false"}
 	nodeArgs          []string
 	tag               string
 	external          = false
@@ -175,7 +175,7 @@ Available clusters:
 	}
 	c.Nodes = nodes
 	c.Secure = secure
-	c.Env = nodeEnv
+	c.Env = strings.Join(nodeEnv, " ")
 	c.Args = nodeArgs
 	if tag != "" {
 		c.Tag = "/" + tag
@@ -1209,7 +1209,7 @@ the 'zfs rollback' command:
 			}
 			fsCmd = `sudo zpool create -f data1 -m /mnt/data1 /dev/sdb`
 		case "ext4":
-			fsCmd = `sudo mkfs.ext4 -F /dev/sdb && sudo mount -o discard,defaults /dev/sdb /mnt/data1`
+			fsCmd = `sudo mkfs.ext4 -F /dev/sdb && sudo mount -o defaults /dev/sdb /mnt/data1`
 		default:
 			return fmt.Errorf("unknown filesystem %q", fs)
 		}
@@ -1777,7 +1777,7 @@ func main() {
 				"start nodes sequentially so node IDs match hostnames")
 			cmd.Flags().StringArrayVarP(
 				&nodeArgs, "args", "a", nil, "node arguments")
-			cmd.Flags().StringVarP(
+			cmd.Flags().StringArrayVarP(
 				&nodeEnv, "env", "e", nodeEnv, "node environment variables")
 			cmd.Flags().StringVarP(
 				&clusterType, "type", "t", clusterType, `cluster type ("cockroach" or "cassandra")`)

pkg/cmd/roachprod/vm/aws/aws.go

@@ -75,12 +75,21 @@ type providerOpts struct {
 	EBSVolumeType      string
 	EBSVolumeSize      int
 	EBSProvisionedIOPs int
+	UseMultipleDisks   bool
+
+	// Use specified ImageAMI when provisioning.
+	// Overrides config.json AMI.
+	ImageAMI string
 
 	// CreateZones stores the list of zones for used cluster creation.
 	// When > 1 zone specified, geo is automatically used, otherwise, geo depends
 	// on the geo flag being set. If no zones specified, defaultCreateZones are
 	// used. See defaultCreateZones.
 	CreateZones []string
+	// CreateRateLimit specifies the rate limit used for aws instance creation.
+	// The request limit from aws' side can vary across regions, as well as the
+	// size of cluster being created.
+	CreateRateLimit float64
 }
 
 const (
@@ -141,6 +150,14 @@ func (o *providerOpts) ConfigureCreateFlags(flags *pflag.FlagSet) {
 			"as AZ:N where N is an integer, the zone will be repeated N times. If > 1\n"+
 			"zone specified, the cluster will be spread out evenly by zone regardless\n"+
 			"of geo (default [%s])", strings.Join(defaultCreateZones, ",")))
+	flags.StringVar(&o.ImageAMI, ProviderName+"-image-ami",
+		"", "Override image AMI to use.  See https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html")
+	flags.BoolVar(&o.UseMultipleDisks, ProviderName+"-enable-multiple-stores",
+		false, "Enable the use of multiple stores by creating one store directory per disk.  Default is to raid0 stripe all disks.")
+	flags.Float64Var(&o.CreateRateLimit, ProviderName+"-create-rate-limit", 2, "aws"+
+		" rate limit (per second) for instance creation. This is used to avoid hitting the request"+
+		" limits from aws, which can vary based on the region, and the size of the cluster being"+
+		" created. Try lowering this limit when hitting 'Request limit exceeded' errors.")
 }
 
 func (o *providerOpts) ConfigureClusterFlags(flags *pflag.FlagSet, _ vm.MultipleProjectsOption) {
@@ -251,8 +268,7 @@ func (p *Provider) Create(names []string, opts vm.CreateOpts) error {
 		}
 	}
 	var g errgroup.Group
-	const rateLimit = 2 // per second
-	limiter := rate.NewLimiter(rateLimit, 2 /* buckets */)
+	limiter := rate.NewLimiter(rate.Limit(p.opts.CreateRateLimit), 2 /* buckets */)
 	for i := range names {
 		capName := names[i]
 		placement := zones[i]
@@ -670,27 +686,35 @@ func (p *Provider) runInstance(name string, zone string, opts vm.CreateOpts) err
 			extraMountOpts = "nobarrier"
 		}
 	}
-	filename, err := writeStartupScript(extraMountOpts)
+	filename, err := writeStartupScript(extraMountOpts, p.opts.UseMultipleDisks)
 	if err != nil {
 		return errors.Wrapf(err, "could not write AWS startup script to temp file")
 	}
 	defer func() {
 		_ = os.Remove(filename)
 	}()
 
+	withFlagOverride := func(cfg string, fl *string) string {
+		if *fl == "" {
+			return cfg
+		}
+		return *fl
+	}
+
 	args := []string{
 		"ec2", "run-instances",
 		"--associate-public-ip-address",
 		"--count", "1",
-		"--image-id", az.region.AMI,
 		"--instance-type", machineType,
+		"--image-id", withFlagOverride(az.region.AMI, &p.opts.ImageAMI),
 		"--key-name", keyName,
 		"--region", az.region.Name,
 		"--security-group-ids", az.region.SecurityGroup,
 		"--subnet-id", az.subnetID,
 		"--tag-specifications", tagSpecs,
 		"--user-data", "file://" + filename,
 	}
+
 	if cpuOptions != "" {
 		args = append(args, "--cpu-options", cpuOptions)
 	}
@@ -702,7 +726,7 @@ func (p *Provider) runInstance(name string, zone string, opts vm.CreateOpts) err
 		case "gp2":
 			ebsParams = fmt.Sprintf("{VolumeSize=%d,VolumeType=%s,DeleteOnTermination=true}",
 				p.opts.EBSVolumeSize, t)
-		case "io1":
+		case "io1", "io2":
 			ebsParams = fmt.Sprintf("{VolumeSize=%d,VolumeType=%s,Iops=%d,DeleteOnTermination=true}",
 				p.opts.EBSVolumeSize, t, p.opts.EBSProvisionedIOPs)
 		default:
@@ -714,7 +738,6 @@ func (p *Provider) runInstance(name string, zone string, opts vm.CreateOpts) err
 			"DeviceName=/dev/sdd,Ebs="+ebsParams,
 		)
 	}
-
 	return p.runJSONCommand(args, &data)
 }
 

pkg/cmd/roachprod/vm/aws/support.go

@@ -39,11 +39,14 @@ set -x
 sudo apt-get update
 sudo apt-get install -qy --no-install-recommends mdadm
 
-mount_opts="discard,defaults"
+mount_opts="defaults"
 {{if .ExtraMountOpts}}mount_opts="${mount_opts},{{.ExtraMountOpts}}"{{end}}
 
+use_multiple_disks='{{if .UseMultipleDisks}}true{{end}}'
+
 disks=()
-mountpoint="/mnt/data1"
+mount_prefix="/mnt/data"
+
 # On different machine types, the drives are either called nvme... or xvdd.
 for d in $(ls /dev/nvme?n1 /dev/xvdd); do
   if ! mount | grep ${d}; then
@@ -53,24 +56,33 @@ for d in $(ls /dev/nvme?n1 /dev/xvdd); do
     echo "Disk ${d} already mounted, skipping..."
   fi
 done
+
+
 if [ "${#disks[@]}" -eq "0" ]; then
+  mountpoint="${mount_prefix}1"
   echo "No disks mounted, creating ${mountpoint}"
   mkdir -p ${mountpoint}
   chmod 777 ${mountpoint}
-elif [ "${#disks[@]}" -eq "1" ]; then
-  echo "One disk mounted, creating ${mountpoint}"
-  mkdir -p ${mountpoint}
-  disk=${disks[0]}
-  mkfs.ext4 -E nodiscard ${disk}
-  mount -o ${mount_opts} ${disk} ${mountpoint}
-  chmod 777 ${mountpoint}
-  echo "${disk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab
+elif [ "${#disks[@]}" -eq "1" ] || [ -n "use_multiple_disks" ]; then
+  disknum=1
+  for disk in "${disks[@]}"
+  do
+    mountpoint="${mount_prefix}${disknum}"
+    disknum=$((disknum + 1 ))
+    echo "Creating ${mountpoint}"
+    mkdir -p ${mountpoint}
+    mkfs.ext4 -F ${disk}
+    mount -o ${mount_opts} ${disk} ${mountpoint}
+    chmod 777 ${mountpoint}
+    echo "${disk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab
+  done
 else
+  mountpoint="${mount_prefix}1"
   echo "${#disks[@]} disks mounted, creating ${mountpoint} using RAID 0"
   mkdir -p ${mountpoint}
   raiddisk="/dev/md0"
   mdadm --create ${raiddisk} --level=0 --raid-devices=${#disks[@]} "${disks[@]}"
-  mkfs.ext4 -E nodiscard ${raiddisk}
+  mkfs.ext4 -F ${raiddisk}
   mount -o ${mount_opts} ${raiddisk} ${mountpoint}
   chmod 777 ${mountpoint}
   echo "${raiddisk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab
@@ -125,12 +137,13 @@ sudo touch /mnt/data1/.roachprod-initialized
 //
 // extraMountOpts, if not empty, is appended to the default mount options. It is
 // a comma-separated list of options for the "mount -o" flag.
-func writeStartupScript(extraMountOpts string) (string, error) {
+func writeStartupScript(extraMountOpts string, useMultiple bool) (string, error) {
 	type tmplParams struct {
-		ExtraMountOpts string
+		ExtraMountOpts   string
+		UseMultipleDisks bool
 	}
 
-	args := tmplParams{ExtraMountOpts: extraMountOpts}
+	args := tmplParams{ExtraMountOpts: extraMountOpts, UseMultipleDisks: useMultiple}
 
 	tmpfile, err := ioutil.TempFile("", "aws-startup-script")
 	if err != nil {