Fetch activity log from azure in e2e tests (Azure#3134)

* Fetch activity log from azure in e2e tests

* Fetch all activity logs from a resource group

* Fetch only failures

test/e2e/azure/cli.go

@@ -358,6 +358,18 @@ func (a *Account) IsClusterExpired(d time.Duration) bool {
 	return age > d
 }
 
+// FetchActivityLog gets all the failures from the activity log for the provided resource group.
+func (a *Account) FetchActivityLog(rg string) (string, error) {
+	var cmd *exec.Cmd
+	cmd = exec.Command("az", "monitor", "activity-log", "list", "--resource-group", rg, "--status", "Failed")
+	util.PrintCommand(cmd)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	return string(out), nil
+}
+
 // CreateStorageAccount will create a new Azure Storage Account
 func (sa *StorageAccount) CreateStorageAccount() error {
 	var cmd *exec.Cmd

test/e2e/runner.go

@@ -131,6 +131,7 @@ func main() {
 			}
 		}
 	} else {
+		cliProvisioner.ResourceGroups = append(rgs, cliProvisioner.Config.Name)
 		engCfg, err := engine.ParseConfig(cfg.CurrentWorkingDir, cfg.ClusterDefinition, cfg.Name)
 		cfg.SetKubeConfig()
 		if err != nil {
@@ -183,31 +184,29 @@ func trap() {
 func teardown() {
 	pt.RecordTotalTime()
 	pt.Write()
+	hostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", cfg.Name, cfg.Location)
+	logsPath := filepath.Join(cfg.CurrentWorkingDir, "_logs", hostname)
+	err := os.MkdirAll(logsPath, 0755)
+	if err != nil {
+		log.Printf("cannot create directory for logs: %s", err)
+	}
+
 	if cliProvisioner.Config.IsKubernetes() && cfg.SoakClusterName == "" {
-		hostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", cfg.Name, cfg.Location)
-		logsPath := filepath.Join(cfg.CurrentWorkingDir, "_logs", hostname)
-		err := os.MkdirAll(logsPath, 0755)
-		if err != nil {
-			log.Printf("cannot create directory for logs: %s", err)
-		}
 		err = cliProvisioner.FetchProvisioningMetrics(logsPath, cfg, acct)
 		if err != nil {
 			log.Printf("cliProvisioner.FetchProvisioningMetrics error: %s\n", err)
 		}
 	}
 	if cliProvisioner.Config.IsOpenShift() {
-		hostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", cfg.Name, cfg.Location)
-		logsPath := filepath.Join(cfg.CurrentWorkingDir, "_logs", hostname)
-		err := os.MkdirAll(logsPath, 0755)
-		if err != nil {
-			log.Printf("cannot create directory for logs: %s", err)
-		}
 		sshKeyPath := cfg.GetSSHKeyPath()
 		adminName := eng.ClusterDefinition.Properties.LinuxProfile.AdminUsername
 		version := eng.Config.OrchestratorVersion
 		distro := eng.Config.Distro
 		outil.FetchOpenShiftLogs(distro, version, sshKeyPath, adminName, cfg.Name, cfg.Location, logsPath)
 	}
+	if err := cliProvisioner.FetchActivityLog(acct, logsPath); err != nil {
+		log.Printf("cannot fetch the activity log: %v", err)
+	}
 	if !cfg.RetainSSH {
 		creds := filepath.Join(cfg.CurrentWorkingDir, "_output/", "*ssh*")
 		files, err := filepath.Glob(creds)

test/e2e/runner/cli_provisioner.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -289,3 +290,18 @@ func (cli *CLIProvisioner) IsPrivate() bool {
 		cli.Engine.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster != nil &&
 		helpers.IsTrueBoolPointer(cli.Engine.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.Enabled)
 }
+
+// FetchActivityLog gets the activity log for the all resource groups used in the provisioner.
+func (cli *CLIProvisioner) FetchActivityLog(acct *azure.Account, logPath string) error {
+	for _, rg := range cli.ResourceGroups {
+		log, err := acct.FetchActivityLog(rg)
+		if err != nil {
+			return fmt.Errorf("cannot fetch activity log for resource group %s: %v", rg, err)
+		}
+		path := filepath.Join(logPath, fmt.Sprintf("activity-log-%s", rg))
+		if err := ioutil.WriteFile(path, []byte(log), 0644); err != nil {
+			return fmt.Errorf("cannot write activity log in file: %v", err)
+		}
+	}
+	return nil
+}