Skip to content

Commit

Permalink
Merge #111926
Browse files Browse the repository at this point in the history 
111926: Roachprod azure add machine r=darrylwong,srosenberg a=smg260

This PR makes several azure specific changes to roachtest and roachprod to support more roachtests.

- Add 96 cpu machine type
- Add `s` series machines for `premium/ultra` disks
- Firewall port configuration for more tests and refactor
- Set a default location from `west` to `west2` for AZ
- `apt-get update` for asyncpg

    
Epic: CC-25185
Release note: none


Co-authored-by: Miral Gadani <[email protected]>
  • Loading branch information
craig[bot] and Miral Gadani committed Oct 10, 2023
2 parents 45882fb + e7c247a commit 77f062e
Show file tree
Hide file tree
Showing 6 changed files with 112 additions and 158 deletions.
16 changes: 13 additions & 3 deletions pkg/cmd/roachtest/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,6 @@ func TestAWSMachineTypeNew(t *testing.T) {

_, _, err2 := spec.SelectAWSMachineTypeNew(16, spec.Low, false, vm.ArchAMD64)
require.Error(t, err2)

_, err3 := spec.SelectAzureMachineType(4, spec.High)
require.Error(t, err3)
}

// TODO(srosenberg): restore the change in https://github.com/cockroachdb/cockroach/pull/111140 after 23.2 branch cut.
Expand Down Expand Up @@ -429,6 +426,19 @@ func TestGCEMachineTypeNew(t *testing.T) {
}
}

func TestAzureMachineType(t *testing.T) {
m, err := spec.SelectAzureMachineType(8, spec.Auto, true)
require.NoError(t, err)
require.Equal(t, "Standard_D8_v3", m)

m, err2 := spec.SelectAzureMachineType(96, spec.Auto, false)
require.NoError(t, err2)
require.Equal(t, "Standard_D96s_v5", m)

_, err3 := spec.SelectAzureMachineType(4, spec.High, true)
require.Error(t, err3)
}

func TestCmdLogFileName(t *testing.T) {
ts := time.Date(2000, 1, 1, 15, 4, 12, 0, time.Local)

Expand Down
2 changes: 1 addition & 1 deletion pkg/cmd/roachtest/spec/cluster_spec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ func (s *ClusterSpec) RoachprodOpts(
case GCE:
machineType, selectedArch = SelectGCEMachineType(s.CPUs, s.Mem, arch)
case Azure:
machineType, err = SelectAzureMachineType(s.CPUs, s.Mem)
machineType, err = SelectAzureMachineType(s.CPUs, s.Mem, s.PreferLocalSSD)
}

if err != nil {
Expand Down
23 changes: 15 additions & 8 deletions pkg/cmd/roachtest/spec/machine_type.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,25 +293,32 @@ func SelectGCEMachineTypeNew(cpus int, mem MemPerCPU, arch vm.CPUArch) (string,

// SelectAzureMachineType selects a machine type given the desired number of CPUs and
// memory per CPU ratio.
func SelectAzureMachineType(cpus int, mem MemPerCPU) (string, error) {
func SelectAzureMachineType(cpus int, mem MemPerCPU, ssd bool) (string, error) {
if mem != Auto && mem != Standard {
return "", errors.Newf("custom memory per CPU not implemented for Azure, memory ratio requested: %d", mem)
}
var premiumStorage string
// If not using Local SSD, the machine type must support premium/ultra storage.
if !ssd {
premiumStorage = "s"
}
switch {
case cpus <= 2:
return "Standard_D2_v3", nil
return fmt.Sprintf("Standard_D2%s_v3", premiumStorage), nil
case cpus <= 4:
return "Standard_D4_v3", nil
return fmt.Sprintf("Standard_D4%s_v3", premiumStorage), nil
case cpus <= 8:
return "Standard_D8_v3", nil
return fmt.Sprintf("Standard_D8%s_v3", premiumStorage), nil
case cpus <= 16:
return "Standard_D16_v3", nil
return fmt.Sprintf("Standard_D16%s_v3", premiumStorage), nil
case cpus <= 36:
return "Standard_D32_v3", nil
return fmt.Sprintf("Standard_D32%s_v3", premiumStorage), nil
case cpus <= 48:
return "Standard_D48_v3", nil
return fmt.Sprintf("Standard_D48%s_v3", premiumStorage), nil
case cpus <= 64:
return "Standard_D64_v3", nil
return fmt.Sprintf("Standard_D64%s_v3", premiumStorage), nil
case cpus <= 96:
return fmt.Sprintf("Standard_D96%s_v5", premiumStorage), nil
default:
return "", errors.Newf("no azure machine type with %d cpus", cpus)
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/cmd/roachtest/tests/asyncpg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@ func registerAsyncpg(r registry.Registry) {
t.Fatal(err)
}

if err := repeatRunE(
ctx, t, c, node, "update apt-get", `sudo apt-get update`,
); err != nil {
t.Fatal(err)
}

if err := repeatRunE(
ctx,
t,
Expand Down
219 changes: 74 additions & 145 deletions pkg/roachprod/vm/azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -854,6 +854,79 @@ func (p *Provider) createNIC(
return
}

// securityRules returns an array of TCP security rules and contains
// a list of well-known, and roachtest specific ports.
func securityRules() *[]network.SecurityRule {
allowTCP := func(name string, priority int32, direction network.SecurityRuleDirection, destPortRange string) network.SecurityRule {
suffix := ""
switch direction {
case network.SecurityRuleDirectionInbound:
suffix = "_Inbound"
case network.SecurityRuleDirectionOutbound:
suffix = "_Outbound"
default:
}
res := network.SecurityRule{
Name: to.StringPtr(name + suffix),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(priority),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: direction,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr(destPortRange),
},
}
return res
}

namedInbound := map[string]string{
"SSH": "22",
"HTTP": "80",
"HTTPS": "43",
"CockroachPG": "26257",
"CockroachAdmin": "26258",
"Grafana": "3000",
"Prometheus": "9090",
"Kafka": "9092",
"WorkloadPPROF": "33333",
"WorkloadPrometheus": "2112-2120",
}

// The names for these are generated in the form Roachtest_<index>_Inbound.
// The mapped roachtests are not exhaustive, and at some point will be
// cumbersome to keep adding exceptions for.
// TODO: (miral) Consider removing all rules if this keeps tripping roachtests.
genericInbound := []string{
"8011", // multitenant
"8081", // backup/*
"9011", // smoketest/secure/multitenan
"9081-9102", // smoketest/secure/multitenant
"20011-20016", //multitenant/upgrade
"27257", //acceptance/gossip/restart-node-one
"27259-27280", // various multitenant tenant SQL ports
"30258", //acceptance/multitenant
}

// The extra 1 is for the single allow all TCP outbound allowTCP.
firewallRules := make([]network.SecurityRule, 1+len(namedInbound)+len(genericInbound))
firewallRules[0] = allowTCP("TCP_All", 300, network.SecurityRuleDirectionOutbound, "*")
r := 1
priority := 300
for ruleName, port := range namedInbound {
firewallRules[r] = allowTCP(ruleName, int32(priority+r), network.SecurityRuleDirectionInbound, port)
r++
}

for i, port := range genericInbound {
firewallRules[r] = allowTCP(fmt.Sprintf("Roachtest_%d", i), int32(priority+r), network.SecurityRuleDirectionInbound, port)
r++
}
return &firewallRules
}

func (p *Provider) getOrCreateNetworkSecurityGroup(
ctx context.Context, name string, resourceGroup resources.Group,
) (network.SecurityGroup, error) {
Expand Down Expand Up @@ -888,151 +961,7 @@ func (p *Provider) getOrCreateNetworkSecurityGroup(

future, err := client.CreateOrUpdate(ctx, *resourceGroup.Name, name, network.SecurityGroup{
SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{
SecurityRules: &[]network.SecurityRule{
{
Name: to.StringPtr("SSH_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(300),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("22"),
},
},
{
Name: to.StringPtr("SSH_Outbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(301),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionOutbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("*"),
},
},
{
Name: to.StringPtr("HTTP_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(320),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("80"),
},
},
{
Name: to.StringPtr("HTTP_Outbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(321),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionOutbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("*"),
},
},
{
Name: to.StringPtr("HTTPS_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(340),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("443"),
},
},
{
Name: to.StringPtr("HTTPS_Outbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(341),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionOutbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("*"),
},
},
{
Name: to.StringPtr("CockroachPG_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(342),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("26257"),
},
},
{
Name: to.StringPtr("CockroachAdmin_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(343),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("26258"),
},
},
{
Name: to.StringPtr("Grafana_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(344),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("3000"),
},
},
{
Name: to.StringPtr("Prometheus_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(345),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("9090"),
},
},
{
Name: to.StringPtr("Kafka_Inbound"),
SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
Priority: to.Int32Ptr(346),
Protocol: network.SecurityRuleProtocolTCP,
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr("*"),
DestinationPortRange: to.StringPtr("9092"),
},
},
},
SecurityRules: securityRules(),
},
Location: resourceGroup.Location,
})
Expand Down
4 changes: 3 additions & 1 deletion pkg/roachprod/vm/azure/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@ type ProviderOpts struct {
DiskCaching string
}

// These default locations support availability zones. At the time of
// this comment, `westus` did not.
var defaultLocations = []string{
"eastus",
"westus",
"westus2",
"westeurope",
}

Expand Down

0 comments on commit 77f062e

Please sign in to comment.