Merge #65941 #66036 #66153 #66192

65941: roachprod: added capability to set OS disk size for AWS, GCP r=rail a=fabiog1901

Added capability in `roachprod` to set the OS/root disk volume size in GB for AWS and GCP.

The global flag is `--os-volume-size`. 

At times you might want to install additional software, or run some simple tests that do not require additional disks.

Usage example:

```
bin/roachprod create fabio-test -n=1 -c aws|gce --os-volume-size=100
```


66036: ui: Adding TransactionsPageConnected for Cluster UI 21.2 r=nathanstilwell a=nathanstilwell

Added a transactionsPageConnected to cluster-ui for use in the published
package. Also added the associated reducer, sagas, and selectors to
support the connection with a Redux store.

I made a slight adjustment with the prop types of transactionsPage for
clarity and to align with other page components.

Release note: None

66153: roachtest: update sqlalchemy roachtest to use SQLA 1.4 r=rafiss a=arulajmani

This was failing because we reorganized the project and bumped the
SQL Alchemy supported version to 1.4.17.

Fixes #66075

Release note: None

66192: sql: enhance readability of builder tests in new schema changer r=ajwerner a=fqazi

Previously, the new schema changer builder tests had readability
issues trying to figure out the elements getting made. This was
inadequate because the builder tests results were hard to read.
To address this, this patch changes the format for the builder
tests to be easier to read.

Release note: None

Co-authored-by: Fabio Ghirardello <[email protected]>
Co-authored-by: Rail Aliiev <[email protected]>
Co-authored-by: Nathan Stilwell <[email protected]>
Co-authored-by: arulajmani <[email protected]>
Co-authored-by: Faizan Qazi <[email protected]>

pkg/cmd/roachprod/main.go

@@ -1857,6 +1857,10 @@ func main() {
 			`Ignored if --local-ssd=false is specified.`)
 	createCmd.Flags().IntVarP(&numNodes,
 		"nodes", "n", 4, "Total number of nodes, distributed across all clouds")
+
+	createCmd.Flags().IntVarP(&createVMOpts.OsVolumeSize,
+		"os-volume-size", "", 10, "OS disk volume size in GB")
+
 	createCmd.Flags().StringSliceVarP(&createVMOpts.VMProviders,
 		"clouds", "c", []string{gce.ProviderName},
 		fmt.Sprintf("The cloud provider(s) to use when creating new vm instances: %s", vm.AllProviderNames()))

pkg/cmd/roachprod/vm/aws/aws.go

@@ -859,25 +859,37 @@ func (p *Provider) runInstance(name string, zone string, opts vm.CreateOpts) err
 			v.Disk = p.opts.DefaultEBSVolume.Disk
 			p.opts.EBSVolumes = append(p.opts.EBSVolumes, v)
 		}
+	}
 
-		mapping, err := json.Marshal(p.opts.EBSVolumes)
-		if err != nil {
-			return err
-		}
+	osDiskVolume := &ebsVolume{
+		DeviceName: "/dev/sda1",
+		Disk: ebsDisk{
+			VolumeType:          defaultEBSVolumeType,
+			VolumeSize:          opts.OsVolumeSize,
+			DeleteOnTermination: true,
+		},
+	}
 
-		deviceMapping, err := ioutil.TempFile("", "aws-block-device-mapping")
-		if err != nil {
-			return err
-		}
-		defer deviceMapping.Close()
-		if _, err := deviceMapping.Write(mapping); err != nil {
-			return err
-		}
-		args = append(args,
-			"--block-device-mapping",
-			"file://"+deviceMapping.Name(),
-		)
+	p.opts.EBSVolumes = append(p.opts.EBSVolumes, osDiskVolume)
+
+	mapping, err := json.Marshal(p.opts.EBSVolumes)
+	if err != nil {
+		return err
+	}
+
+	deviceMapping, err := ioutil.TempFile("", "aws-block-device-mapping")
+	if err != nil {
+		return err
 	}
+	defer deviceMapping.Close()
+	if _, err := deviceMapping.Write(mapping); err != nil {
+		return err
+	}
+	args = append(args,
+		"--block-device-mapping",
+		"file://"+deviceMapping.Name(),
+	)
+
 	return p.runJSONCommand(args, &data)
 }
 

pkg/cmd/roachprod/vm/azure/azure.go

@@ -498,6 +498,11 @@ func (p *Provider) createVM(
 	tags[tagLifetime] = to.StringPtr(opts.Lifetime.String())
 	tags[tagRoachprod] = to.StringPtr("true")
 
+	osVolumeSize := int32(opts.OsVolumeSize)
+	if osVolumeSize < 32 {
+		log.Print("WARNING: increasing the OS volume size to minimally allowed 32GB")
+		osVolumeSize = 32
+	}
 	// Derived from
 	// https://github.com/Azure-Samples/azure-sdk-for-go-samples/blob/79e3f3af791c3873d810efe094f9d61e93a6ccaa/compute/vm.go#L41
 	vm = compute.VirtualMachine{
@@ -520,6 +525,7 @@ func (p *Provider) createVM(
 					ManagedDisk: &compute.ManagedDiskParameters{
 						StorageAccountType: compute.StorageAccountTypesStandardSSDLRS,
 					},
+					DiskSizeGB: to.Int32Ptr(osVolumeSize),
 				},
 			},
 			OsProfile: &compute.OSProfile{

pkg/cmd/roachprod/vm/gce/gcloud.go

@@ -277,7 +277,7 @@ func (o *providerOpts) ConfigureCreateFlags(flags *pflag.FlagSet) {
 		"Image to use to create the vm, "+
 			"use `gcloud compute images list --filter=\"family=ubuntu-2004-lts\"` to list available images")
 
-	flags.IntVar(&o.SSDCount, ProviderName+"-local-ssd-count", 1,
+	flags.IntVar(&o.SSDCount, ProviderName+"-local-ssd-count", 0,
 		"Number of local SSDs to create, only used if local-ssd=true")
 	flags.StringVar(&o.PDVolumeType, ProviderName+"-pd-volume-type", "pd-ssd",
 		"Type of the persistent disk volume, only used if local-ssd=false")
@@ -385,7 +385,6 @@ func (p *Provider) Create(names []string, opts vm.CreateOpts) error {
 		"--scopes", "default,storage-rw",
 		"--image", p.opts.Image,
 		"--image-project", "ubuntu-os-cloud",
-		"--boot-disk-size", "10",
 		"--boot-disk-type", "pd-ssd",
 	}
 
@@ -405,7 +404,7 @@ func (p *Provider) Create(names []string, opts vm.CreateOpts) error {
 		// come in different sizes.
 		// See: https://cloud.google.com/compute/docs/disks/
 		n2MachineTypes := regexp.MustCompile("^[cn]2-.+-16")
-		if n2MachineTypes.MatchString(p.opts.MachineType) && p.opts.SSDCount < 2 {
+		if n2MachineTypes.MatchString(p.opts.MachineType) && p.opts.SSDCount == 1 {
 			fmt.Fprint(os.Stderr, "WARNING: SSD count must be at least 2 for n2 and c2 machine types with 16vCPU. Setting --gce-local-ssd-count to 2.\n")
 			p.opts.SSDCount = 2
 		}
@@ -444,7 +443,7 @@ func (p *Provider) Create(names []string, opts vm.CreateOpts) error {
 
 	args = append(args, "--metadata-from-file", fmt.Sprintf("startup-script=%s", filename))
 	args = append(args, "--project", project)
-
+	args = append(args, fmt.Sprintf("--boot-disk-size=%dGB", opts.OsVolumeSize))
 	var g errgroup.Group
 
 	nodeZones := vm.ZonePlacement(len(zones), len(names))

pkg/cmd/roachprod/vm/vm.go

@@ -140,6 +140,7 @@ type CreateOpts struct {
 		// mounting the SSD. Ignored if UseLocalSSD is not set.
 		NoExt4Barrier bool
 	}
+	OsVolumeSize int
 }
 
 // MultipleProjectsOption is used to specify whether a command accepts multiple

pkg/cmd/roachtest/sqlalchemy.go

@@ -22,7 +22,7 @@ import (
 var sqlAlchemyResultRegex = regexp.MustCompile(`^(?P<test>test.*::.*::[^ \[\]]*(?:\[.*])?) (?P<result>\w+)\s+\[.+]$`)
 var sqlAlchemyReleaseTagRegex = regexp.MustCompile(`^rel_(?P<major>\d+)_(?P<minor>\d+)_(?P<point>\d+)$`)
 
-var supportedSQLAlchemyTag = "rel_1_3_24"
+var supportedSQLAlchemyTag = "rel_1_4_17"
 
 // This test runs the SQLAlchemy dialect test suite against a single Cockroach
 // node.
@@ -155,7 +155,7 @@ func runSQLAlchemy(ctx context.Context, t *test, c *cluster) {
 	// will fail. And it is safe to swallow it here.
 	rawResults, _ := c.RunWithBuffer(ctx, t.l, node,
 		`cd /mnt/data1/sqlalchemy/ && pytest --maxfail=0 \
-		--requirements=cockroachdb.sqlalchemy.test_requirements:Requirements \
+		--requirements=sqlalchemy_cockroachdb.requirements:Requirements \
 		--dburi=cockroachdb://root@localhost:26257/defaultdb?sslmode=disable \
 		test/dialect/test_suite.py
 	`)

pkg/sql/schemachanger/scbuild/builder_test.go

@@ -11,11 +11,13 @@
 package scbuild_test
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	gojson "encoding/json"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -117,29 +119,46 @@ func TestBuilderAlterTable(t *testing.T) {
 	})
 }
 
+// indentText indents text for formatting out marshaled data.
+func indentText(input string, tab string) string {
+	result := strings.Builder{}
+	scanner := bufio.NewScanner(strings.NewReader(input))
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+		result.WriteString(tab)
+		result.WriteString(line)
+		result.WriteString("\n")
+	}
+	return result.String()
+}
+
 // marshalNodes marshals a []*scpb.Node to YAML.
 func marshalNodes(t *testing.T, nodes []*scpb.Node) string {
-	type mapNode struct {
-		Target map[string]interface{}
-		State  string
-	}
-	mapNodes := make([]mapNode, 0, len(nodes))
+	result := strings.Builder{}
 	for _, node := range nodes {
 		var buf bytes.Buffer
-		require.NoError(t, (&jsonpb.Marshaler{}).Marshal(&buf, node.Target))
+		require.NoError(t, (&jsonpb.Marshaler{}).Marshal(&buf, node.Target.Element()))
 
 		target := make(map[string]interface{})
 		require.NoError(t, gojson.Unmarshal(buf.Bytes(), &target))
 
-		mapNodes = append(mapNodes, mapNode{
-			Target: target,
-			State:  node.State.String(),
-		})
+		result.WriteString("- ")
+		result.WriteString(node.Target.Direction.String())
+		result.WriteString(" ")
+		result.WriteString(node.Element().GetAttributes().String())
+		result.WriteString("\n")
+		result.WriteString(indentText(fmt.Sprintf("state: %s\n", node.State.String()), "  "))
+		result.WriteString(indentText("details:\n", "  "))
+		out, err := yaml.Marshal(target)
+		require.NoError(t, err)
+		result.WriteString(indentText(string(out), "    "))
 	}
 
-	out, err := yaml.Marshal(mapNodes)
-	require.NoError(t, err)
-	return string(out)
+	return result.String()
 }
 
 func newTestingBuilderDeps(s serverutils.TestServerInterface) (*scbuild.Dependencies, func()) {