diff --git a/test/e2e/azure/cli.go b/test/e2e/azure/cli.go
index 49c4c9fc96..cd64f4da6f 100644
--- a/test/e2e/azure/cli.go
+++ b/test/e2e/azure/cli.go
@@ -95,9 +95,15 @@ func (a *Account) CreateGroup(name, location string) error {
 	return nil
 }
 
-// DeleteGroup delets a given resource group by name
-func (a *Account) DeleteGroup(name string) error {
-	out, err := exec.Command("az", "group", "delete", "--name", name, "--no-wait", "--yes").CombinedOutput()
+// DeleteGroup deletes a given resource group by name
+func (a *Account) DeleteGroup(name string, wait bool) error {
+	var out []byte
+	var err error
+	if !wait {
+		out, err = exec.Command("az", "group", "delete", "--name", name, "--no-wait", "--yes").CombinedOutput()
+	} else {
+		out, err = exec.Command("az", "group", "delete", "--name", name, "--yes").CombinedOutput()
+	}
 	if err != nil {
 		log.Printf("Error while trying to delete resource group (%s):%s", name, out)
 		return err
diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go
index 5531062e68..50e58c6258 100644
--- a/test/e2e/config/config.go
+++ b/test/e2e/config/config.go
@@ -25,6 +25,7 @@ type Config struct {
 	CleanUpOnExit     bool          `envconfig:"CLEANUP_ON_EXIT" default:"true"`                                        // if set the tests will not clean up rgs when tests finish
 	Timeout           time.Duration `envconfig:"TIMEOUT" default:"10m"`
 	CurrentWorkingDir string
+	SoakClusterName   string `envconfig:"SOAK_CLUSTER_NAME"`
 }
 
 const (
diff --git a/test/e2e/runner.go b/test/e2e/runner.go
index 1d7926e30b..14f9b546c9 100644
--- a/test/e2e/runner.go
+++ b/test/e2e/runner.go
@@ -51,6 +51,11 @@ func main() {
 
 	// Only provision a cluster if there isnt a name present
 	if cfg.Name == "" {
+		if cfg.SoakClusterName != "" {
+			rg := cfg.SoakClusterName
+			log.Printf("Deleting Group:%s\n", rg)
+			acct.DeleteGroup(rg, true)
+		}
 		cliProvisioner, err := runner.BuildCLIProvisioner(cfg, acct, pt)
 		if err != nil {
 			log.Fatalf("Error while trying to build CLI Provisioner:%s", err)
@@ -118,7 +123,7 @@ func teardown() {
 	if cfg.CleanUpOnExit {
 		for _, rg := range rgs {
 			log.Printf("Deleting Group:%s\n", rg)
-			acct.DeleteGroup(rg)
+			acct.DeleteGroup(rg, false)
 		}
 	}
 }
diff --git a/test/e2e/runner/cli_provisioner.go b/test/e2e/runner/cli_provisioner.go
index ec7ad0f0b5..29c591bb4b 100644
--- a/test/e2e/runner/cli_provisioner.go
+++ b/test/e2e/runner/cli_provisioner.go
@@ -57,7 +57,7 @@ func (cli *CLIProvisioner) Run() error {
 				cli.Point.RecordProvisionError()
 			} else if i == cli.ProvisionRetries {
 				cli.Point.RecordProvisionError()
-				return fmt.Errorf("Exceeded provision retry count")
+				return fmt.Errorf("Exceeded provision retry count: %s", err)
 			}
 		} else {
 			cli.Point.RecordProvisionSuccess()
@@ -76,17 +76,22 @@ func (cli *CLIProvisioner) Run() error {
 
 func (cli *CLIProvisioner) provision() error {
 	cli.Config.Name = cli.generateName()
+	if cli.Config.SoakClusterName != "" {
+		cli.Config.Name = cli.Config.SoakClusterName
+	}
 	os.Setenv("NAME", cli.Config.Name)
 	log.Printf("Cluster name:%s\n", cli.Config.Name)
 
 	outputPath := filepath.Join(cli.Config.CurrentWorkingDir, "_output")
 	os.Mkdir(outputPath, 0755)
 
-	out, err := exec.Command("ssh-keygen", "-f", cli.Config.GetSSHKeyPath(), "-q", "-N", "", "-b", "2048", "-t", "rsa").CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("Error while trying to generate ssh key:%s\nOutput:%s", err, out)
+	if cli.Config.SoakClusterName == "" {
+		out, err := exec.Command("ssh-keygen", "-f", cli.Config.GetSSHKeyPath(), "-q", "-N", "", "-b", "2048", "-t", "rsa").CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("Error while trying to generate ssh key:%s\nOutput:%s", err, out)
+		}
+		exec.Command("chmod", "0600", cli.Config.GetSSHKeyPath()+"*")
 	}
-	exec.Command("chmod", "0600", cli.Config.GetSSHKeyPath()+"*")
 
 	publicSSHKey, err := cli.Config.ReadPublicSSHKey()
 	if err != nil {