Skip to content

Commit

Permalink
Automate proxy checks for release testing
Browse files Browse the repository at this point in the history
Co-authored-by: Markus Walker <[email protected]>
Co-authored-by: Josh Keslar <[email protected]>
  • Loading branch information
markusewalker and Josh-Diamond committed Jan 29, 2025
1 parent 57fac91 commit 68c0307
Show file tree
Hide file tree
Showing 44 changed files with 1,363 additions and 84 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/verify-changes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ jobs:
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
uses: actions/checkout@v3
with:
submodules: recursive

- name: Setup Go
uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
uses: actions/setup-go@v3
with:
go-version-file: './go.mod'

Expand All @@ -36,7 +36,7 @@ jobs:
run: ./.github/scripts/build-packages.sh

- name: Golangci Lint
uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
uses: golangci/golangci-lint-action@v3
with:
# Patch version isn't needed; https://github.com/golangci/golangci-lint-action?tab=readme-ov-file#internals
version: v1.57
Expand Down
2 changes: 2 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ ARG RKE_KEY_PATH
ARG SANITY_KEY_PATH
ARG AIRGAP_KEY_PATH
ARG REGISTRY_KEY_PATH
ARG PROXY_KEY_PATH

ENV QASE_TEST_RUN_ID=${QASE_TEST_RUN_ID}
ENV TERRAFORM_VERSION=${TERRAFORM_VERSION}
Expand All @@ -40,6 +41,7 @@ ENV RKE_KEY_PATH=${RKE_KEY_PATH}
ENV SANITY_KEY_PATH=${SANITY_KEY_PATH}
ENV AIRGAP_KEY_PATH=${AIRGAP_KEY_PATH}
ENV REGISTRY_KEY_PATH=${REGISTRY_KEY_PATH}
ENV PROXY_KEY_PATH=${PROXY_KEY_PATH}

RUN wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -q && apt-get update > /dev/null && apt-get install unzip > /dev/null && \
unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip > /dev/null && \
Expand Down
4 changes: 4 additions & 0 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ RKE_KEY_PATH="${RKE_KEY_PATH:-}"
SANITY_KEY_PATH="${SANITY_KEY_PATH:-}"
AIRGAP_KEY_PATH="${AIRGAP_KEY_PATH:-}"
REGISTRY_KEY_PATH="${REGISTRY_KEY_PATH:-}"
PROXY_KEY_PATH="${PROXY_KEY_PATH:-}"
UPGRADE_KEY_PATH="${UPGRADE_KEY_PATH:-}"

TRIM_JOB_NAME=$(basename "$JOB_NAME")

Expand All @@ -37,6 +39,8 @@ while [[ 3 -gt $count ]]; do
--build-arg SANITY_KEY_PATH="$SANITY_KEY_PATH" \
--build-arg AIRGAP_KEY_PATH="$AIRGAP_KEY_PATH" \
--build-arg REGISTRY_KEY_PATH="$REGISTRY_KEY_PATH" \
--build-arg PROXY_KEY_PATH="$PROXY_KEY_PATH" \
--build-arg UPGRADE_KEY_PATH="$UPGRADE_KEY_PATH" \
--build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \
--build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" \
-t tfp-automation-validation-"${TRIM_JOB_NAME}""${BUILD_NUMBER}"
Expand Down
37 changes: 23 additions & 14 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ type Nodepool struct {
MaxPodsContraint int64 `json:"maxPodsContraint,omitempty" yaml:"maxPodsContraint,omitempty"`
}

type Proxy struct {
ProxyBastion string `json:"proxyBastion,omitempty" yaml:"proxyBastion,omitempty"`
}

type PrivateRegistries struct {
AuthConfigSecretName string `json:"authConfigSecretName,omitempty" yaml:"authConfigSecretName,omitempty"`
CABundle string `json:"caBundle,omitempty" yaml:"caBundle,omitempty"`
Expand All @@ -108,20 +112,24 @@ type PrivateRegistries struct {
}

type Standalone struct {
AirgapInternalFQDN string `json:"airgapInternalFQDN,omitempty" yaml:"airgapInternalFQDN,omitempty"`
BootstrapPassword string `json:"bootstrapPassword,omitempty" yaml:"bootstrapPassword,omitempty"`
CertManagerVersion string `json:"certManagerVersion,omitempty" yaml:"certManagerVersion,omitempty"`
RancherChartVersion string `json:"rancherChartVersion,omitempty" yaml:"rancherChartVersion,omitempty"`
RancherChartRepository string `json:"rancherChartRepository,omitempty" yaml:"rancherChartRepository,omitempty"`
RancherHostname string `json:"rancherHostname,omitempty" yaml:"rancherHostname,omitempty"`
RancherImage string `json:"rancherImage,omitempty" yaml:"rancherImage,omitempty"`
RancherRepo string `json:"rancherRepo,omitempty" yaml:"rancherRepo,omitempty"`
RancherTagVersion string `json:"rancherTagVersion,omitempty" yaml:"rancherTagVersion,omitempty"`
OSUser string `json:"osUser,omitempty" yaml:"osUser,omitempty"`
OSGroup string `json:"osGroup,omitempty" yaml:"osGroup,omitempty"`
RKE2Version string `json:"rke2Version,omitempty" yaml:"rke2Version,omitempty"`
StagingRancherAgentImage string `json:"stagingRancherAgentImage,omitempty" yaml:"stagingRancherAgentImage,omitempty"`
Type string `json:"type,omitempty" yaml:"type,omitempty"`
AirgapInternalFQDN string `json:"airgapInternalFQDN,omitempty" yaml:"airgapInternalFQDN,omitempty"`
BootstrapPassword string `json:"bootstrapPassword,omitempty" yaml:"bootstrapPassword,omitempty"`
CertManagerVersion string `json:"certManagerVersion,omitempty" yaml:"certManagerVersion,omitempty"`
ProxyRancher bool `json:"proxyRancher,omitempty" yaml:"proxyRancher,omitempty"`
RancherChartRepository string `json:"rancherChartRepository,omitempty" yaml:"rancherChartRepository,omitempty"`
RancherHostname string `json:"rancherHostname,omitempty" yaml:"rancherHostname,omitempty"`
RancherImage string `json:"rancherImage,omitempty" yaml:"rancherImage,omitempty"`
RancherTagVersion string `json:"rancherTagVersion,omitempty" yaml:"rancherTagVersion,omitempty"`
Repo string `json:"repo,omitempty" yaml:"repo,omitempty"`
OSUser string `json:"osUser,omitempty" yaml:"osUser,omitempty"`
OSGroup string `json:"osGroup,omitempty" yaml:"osGroup,omitempty"`
RKE2Version string `json:"rke2Version,omitempty" yaml:"rke2Version,omitempty"`
StagingRancherAgentImage string `json:"stagingRancherAgentImage,omitempty" yaml:"stagingRancherAgentImage,omitempty"`
UpgradedRancherChartRepository string `json:"upgradedRancherChartRepository,omitempty" yaml:"upgradedRancherChartRepository,omitempty"`
UpgradedRancherImage string `json:"upgradedRancherImage,omitempty" yaml:"upgradedRancherImage,omitempty"`
UpgradedRancherStagingImage string `json:"upgradedRancherStagingImage,omitempty" yaml:"upgradedRancherStagingImage,omitempty"`
UpgradedRancherRepo string `json:"upgradedRancherRepo,omitempty" yaml:"upgradedRancherRepo,omitempty"`
UpgradedRancherTagVersion string `json:"upgradedRancherTagVersion,omitempty" yaml:"upgradedRancherTagVersion,omitempty"`
}

type StandaloneRegistry struct {
Expand Down Expand Up @@ -166,6 +174,7 @@ type TerraformConfig struct {
NodeTemplateName string `json:"nodeTemplateName,omitempty" yaml:"nodeTemplateName,omitempty"`
PrivateKeyPath string `json:"privateKeyPath,omitempty" yaml:"privateKeyPath,omitempty"`
PrivateRegistries *PrivateRegistries `json:"privateRegistries,omitempty" yaml:"privateRegistries,omitempty"`
Proxy *Proxy `json:"proxy,omitempty" yaml:"proxy,omitempty"`
Standalone *Standalone `json:"standalone,omitempty" yaml:"standalone,omitempty"`
StandaloneRegistry *StandaloneRegistry `json:"standaloneRegistry,omitempty" yaml:"standaloneRegistry,omitempty"`
}
Expand Down
2 changes: 2 additions & 0 deletions framework/set/defaults/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const (
ResourceName = "name"
Namespace = "namespace"
Triggers = "triggers"
Value = "value"

DependsOn = "depends_on"
GenerateName = "generate_name"
Expand All @@ -26,6 +27,7 @@ const (
ClusterV2 = "rancher2_cluster_v2"
SecretV2 = "rancher2_secret_v2"

AgentEnvVars = "agent_env_vars"
RkeConfig = "rke_config"
KubernetesVersion = "kubernetes_version"
Network = "network"
Expand Down
6 changes: 0 additions & 6 deletions framework/set/provisioning/airgap/setConfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/rancher/tfp-automation/framework/set/provisioning/airgap/nullresource"
"github.com/rancher/tfp-automation/framework/set/provisioning/custom/locals"
v2 "github.com/rancher/tfp-automation/framework/set/provisioning/custom/rke2k3s"
rke2k3s "github.com/rancher/tfp-automation/framework/set/provisioning/nodedriver/rke2k3s"
airgap "github.com/rancher/tfp-automation/framework/set/resources/airgap/aws"
"github.com/rancher/tfp-automation/framework/set/resources/sanity/aws"
"github.com/sirupsen/logrus"
Expand All @@ -32,11 +31,6 @@ func SetAirgapRKE2K3s(rancherConfig *rancher.Config, terraformConfig *config.Ter
v2.SetRancher2ClusterV2(rootBody, terraformConfig, terratestConfig, clusterName)
rootBody.AppendNewline()

if terraformConfig.StandaloneRegistry.Authenticated {
rke2k3s.CreateRegistrySecret(terraformConfig, clusterName, rootBody)
rootBody.AppendNewline()
}

aws.CreateAWSInstances(rootBody, terraformConfig, terratestConfig, bastion)
rootBody.AppendNewline()

Expand Down
5 changes: 5 additions & 0 deletions framework/set/provisioning/nodedriver/rke1/setConfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/rancher/tfp-automation/config"
"github.com/rancher/tfp-automation/defaults/modules"
"github.com/rancher/tfp-automation/framework/set/defaults"
v2 "github.com/rancher/tfp-automation/framework/set/provisioning/nodedriver/rke2k3s"
aws "github.com/rancher/tfp-automation/framework/set/provisioning/providers/aws"
azure "github.com/rancher/tfp-automation/framework/set/provisioning/providers/azure"
linode "github.com/rancher/tfp-automation/framework/set/provisioning/providers/linode"
Expand Down Expand Up @@ -97,6 +98,10 @@ func SetRKE1(terraformConfig *config.TerraformConfig, clusterName, poolName, k8s
clusterBlockBody.SetAttributeValue(defaults.ResourceName, cty.StringVal(clusterName))
clusterBlockBody.SetAttributeValue(defaults.DefaultPodSecurityAdmission, cty.StringVal(psact))

if terraformConfig.Proxy.ProxyBastion != "" {
v2.SetProxyConfig(clusterBlockBody, terraformConfig)
}

rkeConfigBlock := clusterBlockBody.AppendNewBlock(defaults.RkeConfig, nil)
rkeConfigBlockBody := rkeConfigBlock.Body()

Expand Down
9 changes: 9 additions & 0 deletions framework/set/provisioning/nodedriver/rke2k3s/setConfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ const (
systemDefaultRegistry = "system-default-registry"
project = "project"
endpoints = "endpoints"

httpProxy = "HTTP_PROXY"
httpsProxy = "HTTPS_PROXY"
noProxy = "NO_PROXY"
noProxyValue = "localhost,127.0.0.0/8,10.0.0.0/8,172.0.0.0/8,192.168.0.0/16,.svc,.cluster.local,cattle-system.svc,169.254.169.254"
)

// SetRKE2K3s is a function that will set the RKE2/K3S configurations in the main.tf file.
Expand Down Expand Up @@ -111,6 +116,10 @@ func SetRKE2K3s(client *rancher.Client, terraformConfig *config.TerraformConfig,
clusterBlockBody.SetAttributeValue(defaults.DefaultPodSecurityAdmission, cty.StringVal(psact))
clusterBlockBody.SetAttributeValue(defaults.DefaultClusterRoleForProjectMembers, cty.StringVal(terraformConfig.DefaultClusterRoleForProjectMembers))

if terraformConfig.Proxy.ProxyBastion != "" {
SetProxyConfig(clusterBlockBody, terraformConfig)
}

rkeConfigBlock := clusterBlockBody.AppendNewBlock(defaults.RkeConfig, nil)
rkeConfigBlockBody := rkeConfigBlock.Body()

Expand Down
31 changes: 31 additions & 0 deletions framework/set/provisioning/nodedriver/rke2k3s/setProxyConfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package rke2k3s

import (
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/rancher/tfp-automation/config"
"github.com/rancher/tfp-automation/framework/set/defaults"
"github.com/zclconf/go-cty/cty"
)

// SetProxyConfig is a function that will set the proxy configurations in the main.tf file.
func SetProxyConfig(clusterBlockBody *hclwrite.Body, terraformConfig *config.TerraformConfig) error {
agentVarsOneBlock := clusterBlockBody.AppendNewBlock(defaults.AgentEnvVars, nil)
agentVarsOneBlockBody := agentVarsOneBlock.Body()

agentVarsOneBlockBody.SetAttributeValue(defaults.ResourceName, cty.StringVal(httpProxy))
agentVarsOneBlockBody.SetAttributeValue(defaults.Value, cty.StringVal("http://"+terraformConfig.Proxy.ProxyBastion+":3128"))

agentVarsTwoBlock := clusterBlockBody.AppendNewBlock(defaults.AgentEnvVars, nil)
agentVarsTwoBlockBody := agentVarsTwoBlock.Body()

agentVarsTwoBlockBody.SetAttributeValue(defaults.ResourceName, cty.StringVal(httpsProxy))
agentVarsTwoBlockBody.SetAttributeValue(defaults.Value, cty.StringVal("http://"+terraformConfig.Proxy.ProxyBastion+":3128"))

agentVarsThreeBlock := clusterBlockBody.AppendNewBlock(defaults.AgentEnvVars, nil)
agentVarsThreeBlockBody := agentVarsThreeBlock.Body()

agentVarsThreeBlockBody.SetAttributeValue(defaults.ResourceName, cty.StringVal(noProxy))
agentVarsThreeBlockBody.SetAttributeValue(defaults.Value, cty.StringVal(noProxyValue))

return nil
}
27 changes: 13 additions & 14 deletions framework/set/resources/airgap/rancher/setup.sh
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
#!/bin/bash

RANCHER_REPO=$1
RANCHER_CHART_REPO=$2
TYPE=$3
CERT_MANAGER_VERSION=$4
HOSTNAME=$5
INTERNAL_FQDN=$6
RANCHER_TAG_VERSION=$7
BOOTSTRAP_PASSWORD=$8
RANCHER_IMAGE=$9
STAGING_RANCHER_AGENT_IMAGE=${10}
REGISTRY=${11}
RANCHER_CHART_REPO=$1
REPO=$2
CERT_MANAGER_VERSION=$3
HOSTNAME=$4
INTERNAL_FQDN=$5
RANCHER_TAG_VERSION=$6
BOOTSTRAP_PASSWORD=$7
RANCHER_IMAGE=$8
STAGING_RANCHER_AGENT_IMAGE=${9}
REGISTRY=${10}

set -ex

Expand All @@ -21,7 +20,7 @@ chmod +x get_helm.sh
rm get_helm.sh

echo "Adding Helm chart repo"
helm repo add ${RANCHER_REPO} ${RANCHER_CHART_REPO}${TYPE}
helm repo add rancher-${REPO} ${RANCHER_CHART_REPO}${REPO}

echo "Installing cert manager"
kubectl create ns cattle-system
Expand All @@ -36,7 +35,7 @@ sleep 60

echo "Installing Rancher"
if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
helm upgrade --install rancher rancher-${REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
--set hostname=${HOSTNAME} \
--set rancherImageTag=${RANCHER_TAG_VERSION} \
--set rancherImage=${REGISTRY}/${RANCHER_IMAGE} \
Expand All @@ -45,7 +44,7 @@ if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
--set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel

else
helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
helm upgrade --install rancher rancher-${REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
--set hostname=${HOSTNAME} \
--set rancherImage=${RANCHER_IMAGE} \
--set rancherImageTag=${RANCHER_TAG_VERSION} \
Expand Down
4 changes: 2 additions & 2 deletions framework/set/resources/airgap/rancher/setupAirgapRancher.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ func CreateAirgapRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwri

_, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, installRancher)

command := "bash -c '/tmp/setup.sh " + terraformConfig.Standalone.RancherRepo + " " + terraformConfig.Standalone.RancherChartRepository + " " +
terraformConfig.Standalone.Type + " " + terraformConfig.Standalone.CertManagerVersion + " " +
command := "bash -c '/tmp/setup.sh " + terraformConfig.Standalone.RancherChartRepository + " " +
terraformConfig.Standalone.Repo + " " + terraformConfig.Standalone.CertManagerVersion + " " +
terraformConfig.Standalone.RancherHostname + " " + " " + terraformConfig.Standalone.AirgapInternalFQDN + " " +
terraformConfig.Standalone.RancherTagVersion + " " + terraformConfig.Standalone.BootstrapPassword + " " +
terraformConfig.Standalone.RancherImage
Expand Down
86 changes: 86 additions & 0 deletions framework/set/resources/proxy/createMainTF.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package proxy

import (
"os"
"testing"

"github.com/gruntwork-io/terratest/modules/terraform"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/rancher/tfp-automation/config"
"github.com/rancher/tfp-automation/framework/set/resources/proxy/rancher"
"github.com/rancher/tfp-automation/framework/set/resources/proxy/rke2"
"github.com/rancher/tfp-automation/framework/set/resources/proxy/squid"
"github.com/rancher/tfp-automation/framework/set/resources/sanity"
"github.com/rancher/tfp-automation/framework/set/resources/sanity/aws"
)

const (
rke2Bastion = "rke2_bastion"
rke2ServerOne = "rke2_server1"
rke2ServerTwo = "rke2_server2"
rke2ServerThree = "rke2_server3"

rke2ServerOnePublicDNS = "rke2_server1_public_dns"
rke2BastionPublicDNS = "rke2_bastion_public_dns"
rke2ServerOnePrivateIP = "rke2_server1_private_ip"
rke2ServerTwoPublicDNS = "rke2_server2_public_dns"
rke2ServerThreePublicDNS = "rke2_server3_public_dns"

terraformConst = "terraform"
)

// CreateMainTF is a helper function that will create the main.tf file for creating a Rancher server behind a proxy.
func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath string, terraformConfig *config.TerraformConfig,
terratest *config.TerratestConfig) (string, string, error) {
var file *os.File
file = sanity.OpenFile(file, keyPath)
defer file.Close()

newFile := hclwrite.NewEmptyFile()
rootBody := newFile.Body()

tfBlock := rootBody.AppendNewBlock(terraformConst, nil)
tfBlockBody := tfBlock.Body()

instances := []string{rke2Bastion, rke2ServerOne, rke2ServerTwo, rke2ServerThree}
file, err := aws.CreateAWSResources(file, newFile, tfBlockBody, rootBody, terraformConfig, terratest, instances)
if err != nil {
return "", "", err
}

terraform.InitAndApply(t, terraformOptions)

rke2BastionPublicDNS := terraform.Output(t, terraformOptions, rke2BastionPublicDNS)
rke2ServerOnePublicDNS := terraform.Output(t, terraformOptions, rke2ServerOnePublicDNS)
rke2ServerOnePrivateIP := terraform.Output(t, terraformOptions, rke2ServerOnePrivateIP)
rke2ServerTwoPublicDNS := terraform.Output(t, terraformOptions, rke2ServerTwoPublicDNS)
rke2ServerThreePublicDNS := terraform.Output(t, terraformOptions, rke2ServerThreePublicDNS)

terraform.InitAndApply(t, terraformOptions)

file = sanity.OpenFile(file, keyPath)
file, err = squid.CreateSquidProxy(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS)
if err != nil {
return "", "", err
}

terraform.InitAndApply(t, terraformOptions)

file = sanity.OpenFile(file, keyPath)
file, err = rke2.CreateRKE2Cluster(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS)
if err != nil {
return "", "", err
}

terraform.InitAndApply(t, terraformOptions)

file = sanity.OpenFile(file, keyPath)
file, err = rancher.CreateProxiedRancher(file, newFile, rootBody, terraformConfig, rke2ServerOnePublicDNS, rke2BastionPublicDNS)
if err != nil {
return "", "", err
}

terraform.InitAndApply(t, terraformOptions)

return rke2BastionPublicDNS, rke2ServerOnePublicDNS, nil
}
Loading

0 comments on commit 68c0307

Please sign in to comment.