Handle errors in integration test setups

Thanks @kev-the-dev Closes juanfont#1460 Signed-off-by: Kristoffer Dalby <[email protected]>
kradalby · Aug 29, 2023 · 72290e2 · 72290e2
1 parent fb203a2
commit 72290e2
Show file tree

Hide file tree

Showing 11 changed files with 184 additions and 146 deletions.
diff --git a/cmd/headscale/headscale.go b/cmd/headscale/headscale.go
@@ -48,7 +48,7 @@ func main() {
 
 	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
 	log.Logger = log.Output(zerolog.ConsoleWriter{
-		Out:        os.Stdout,
+		Out:        os.Stderr,
 		TimeFormat: time.RFC3339,
 		NoColor:    !colors,
 	})

diff --git a/integration/auth_oidc_test.go b/integration/auth_oidc_test.go
@@ -171,7 +171,10 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) {
 	t.Logf("%d successful pings out of %d (before expiry)", success, len(allClients)*len(allIps))
 
 	// await all nodes being logged out after OIDC token expiry
-	scenario.WaitForTailscaleLogout()
+	err = scenario.WaitForTailscaleLogout()
+	if err != nil {
+		t.Errorf("failed to log out tailscale nodes: %s", err)
+	}
 
 	err = scenario.Shutdown()
 	if err != nil {
@@ -188,7 +191,7 @@ func (s *AuthOIDCScenario) CreateHeadscaleEnv(
 		return err
 	}
 
-	err = headscale.WaitForReady()
+	err = headscale.WaitForRunning()
 	if err != nil {
 		return err
 	}
@@ -311,13 +314,9 @@ func (s *AuthOIDCScenario) runTailscaleUp(
 	log.Printf("running tailscale up for user %s", userStr)
 	if user, ok := s.users[userStr]; ok {
 		for _, client := range user.Clients {
-			user.joinWaitGroup.Add(1)
-
-			go func(c TailscaleClient) {
-				defer user.joinWaitGroup.Done()
-
-				// TODO(juanfont): error handle this
-				loginURL, err := c.UpWithLoginURL(loginServer)
+			c := client
+			user.joinWaitGroup.Go(func() error {
+				loginURL, err := c.LoginWithURL(loginServer)
 				if err != nil {
 					log.Printf("failed to run tailscale up: %s", err)
 				}
@@ -336,9 +335,14 @@ func (s *AuthOIDCScenario) runTailscaleUp(
 				req, _ := http.NewRequestWithContext(ctx, http.MethodGet, loginURL.String(), nil)
 				resp, err := httpClient.Do(req)
 				if err != nil {
-					log.Printf("%s failed to get login url %s: %s", c.Hostname(), loginURL, err)
-
-					return
+					log.Printf(
+						"%s failed to get login url %s: %s",
+						c.Hostname(),
+						loginURL,
+						err,
+					)
+
+					return err
 				}
 
 				defer resp.Body.Close()
@@ -347,24 +351,23 @@ func (s *AuthOIDCScenario) runTailscaleUp(
 				if err != nil {
 					log.Printf("%s failed to read response body: %s", c.Hostname(), err)
 
-					return
+					return err
 				}
 
 				log.Printf("Finished request for %s to join tailnet", c.Hostname())
-			}(client)
 
-			err = client.WaitForReady()
-			if err != nil {
-				log.Printf("error waiting for client %s to be ready: %s", client.Hostname(), err)
-			}
+				return nil
+			})
 
 			log.Printf("client %s is ready", client.Hostname())
 		}
 
-		user.joinWaitGroup.Wait()
+		if err := user.joinWaitGroup.Wait(); err != nil {
+			return err
+		}
 
 		for _, client := range user.Clients {
-			err := client.WaitForReady()
+			err := client.WaitForRunning()
 			if err != nil {
 				log.Printf("client %s was not ready: %s", client.Hostname(), err)
 

diff --git a/integration/auth_web_flow_test.go b/integration/auth_web_flow_test.go
@@ -134,7 +134,10 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 		}
 	}
 
-	scenario.WaitForTailscaleLogout()
+	err = scenario.WaitForTailscaleLogout()
+	if err != nil {
+		t.Errorf("failed to log out tailscale nodes: %s", err)
+	}
 
 	t.Logf("all clients logged out")
 
@@ -218,7 +221,7 @@ func (s *AuthWebFlowScenario) CreateHeadscaleEnv(
 		return err
 	}
 
-	err = headscale.WaitForReady()
+	err = headscale.WaitForRunning()
 	if err != nil {
 		return err
 	}
@@ -250,32 +253,37 @@ func (s *AuthWebFlowScenario) runTailscaleUp(
 	log.Printf("running tailscale up for user %s", userStr)
 	if user, ok := s.users[userStr]; ok {
 		for _, client := range user.Clients {
-			user.joinWaitGroup.Add(1)
-
-			go func(c TailscaleClient) {
-				defer user.joinWaitGroup.Done()
-
-				// TODO(juanfont): error handle this
-				loginURL, err := c.UpWithLoginURL(loginServer)
+			c := client
+			user.joinWaitGroup.Go(func() error {
+				loginURL, err := c.LoginWithURL(loginServer)
 				if err != nil {
-					log.Printf("failed to run tailscale up: %s", err)
+					log.Printf("failed to run tailscale up (%s): %s", c.Hostname(), err)
+
+					return err
 				}
 
 				err = s.runHeadscaleRegister(userStr, loginURL)
 				if err != nil {
-					log.Printf("failed to register client: %s", err)
+					log.Printf("failed to register client (%s): %s", c.Hostname(), err)
+
+					return err
 				}
-			}(client)
 
-			err := client.WaitForReady()
+				return nil
+			})
+
+			err := client.WaitForRunning()
 			if err != nil {
 				log.Printf("error waiting for client %s to be ready: %s", client.Hostname(), err)
 			}
 		}
-		user.joinWaitGroup.Wait()
+
+		if err := user.joinWaitGroup.Wait(); err != nil {
+			return err
+		}
 
 		for _, client := range user.Clients {
-			err := client.WaitForReady()
+			err := client.WaitForRunning()
 			if err != nil {
 				log.Printf("client %s was not ready: %s", client.Hostname(), err)
 

diff --git a/integration/control.go b/integration/control.go
@@ -13,7 +13,7 @@ type ControlServer interface {
 	ConnectToNetwork(network *dockertest.Network) error
 	GetHealthEndpoint() string
 	GetEndpoint() string
-	WaitForReady() error
+	WaitForRunning() error
 	CreateUser(user string) error
 	CreateAuthKey(user string, reusable bool, ephemeral bool) (*v1.PreAuthKey, error)
 	ListMachinesInUser(user string) ([]*v1.Machine, error)

diff --git a/integration/embedded_derp_test.go b/integration/embedded_derp_test.go
@@ -2,7 +2,6 @@ package integration
 
 import (
 	"fmt"
-	"log"
 	"net/url"
 	"testing"
 
@@ -105,7 +104,7 @@ func (s *EmbeddedDERPServerScenario) CreateHeadscaleEnv(
 
 	headscaleURL.Host = fmt.Sprintf("%s:%s", hsServer.GetHostname(), headscaleURL.Port())
 
-	err = hsServer.WaitForReady()
+	err = hsServer.WaitForRunning()
 	if err != nil {
 		return err
 	}
@@ -186,37 +185,35 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
 
 			cert := hsServer.GetCert()
 
-			user.createWaitGroup.Add(1)
-
 			opts = append(opts,
 				tsic.WithHeadscaleTLS(cert),
 			)
 
-			go func() {
-				defer user.createWaitGroup.Done()
-
-				// TODO(kradalby): error handle this
+			user.createWaitGroup.Go(func() error {
 				tsClient, err := tsic.New(
 					s.pool,
 					version,
 					network,
 					opts...,
 				)
 				if err != nil {
-					// return fmt.Errorf("failed to add tailscale node: %w", err)
-					log.Printf("failed to create tailscale node: %s", err)
+					return fmt.Errorf("failed to create tailscale node: %w", err)
 				}
 
-				err = tsClient.WaitForReady()
+				err = tsClient.WaitForRunning()
 				if err != nil {
-					// return fmt.Errorf("failed to add tailscale node: %w", err)
-					log.Printf("failed to wait for tailscaled: %s", err)
+					return fmt.Errorf("failed to wait for tailscaled to need login: %w", err)
 				}
 
 				user.Clients[tsClient.Hostname()] = tsClient
-			}()
+
+				return nil
+			})
+		}
+
+		if err := user.createWaitGroup.Wait(); err != nil {
+			return err
 		}
-		user.createWaitGroup.Wait()
 
 		return nil
 	}

diff --git a/integration/general_test.go b/integration/general_test.go
@@ -26,8 +26,12 @@ func TestPingAllByIP(t *testing.T) {
 	}
 
 	spec := map[string]int{
+		// "user1": 40,
 		"user1": len(TailscaleVersions),
 		"user2": len(TailscaleVersions),
+		"user3": len(TailscaleVersions),
+		"user4": len(TailscaleVersions),
+		"user5": len(TailscaleVersions),
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyip"))
@@ -108,7 +112,10 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) {
 		}
 	}
 
-	scenario.WaitForTailscaleLogout()
+	err = scenario.WaitForTailscaleLogout()
+	if err != nil {
+		t.Errorf("failed to log out tailscale nodes: %s", err)
+	}
 
 	t.Logf("all clients logged out")
 
@@ -261,7 +268,10 @@ func TestEphemeral(t *testing.T) {
 		}
 	}
 
-	scenario.WaitForTailscaleLogout()
+	err = scenario.WaitForTailscaleLogout()
+	if err != nil {
+		t.Errorf("failed to log out tailscale nodes: %s", err)
+	}
 
 	t.Logf("all clients logged out")
 
@@ -488,7 +498,7 @@ func TestResolveMagicDNS(t *testing.T) {
 	spec := map[string]int{
 		// Omit 1.16.2 (-1) because it does not have the FQDN field
 		"magicdns1": len(TailscaleVersions) - 1,
-		"magicdns2": len(TailscaleVersions) - 1,
+		// "magicdns2": len(TailscaleVersions) - 1,
 	}
 
 	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("magicdns"))
@@ -553,10 +563,10 @@ func TestResolveMagicDNS(t *testing.T) {
 		}
 	}
 
-	err = scenario.Shutdown()
-	if err != nil {
-		t.Errorf("failed to tear down scenario: %s", err)
-	}
+	// err = scenario.Shutdown()
+	// if err != nil {
+	// 	t.Errorf("failed to tear down scenario: %s", err)
+	// }
 }
 
 func TestExpireNode(t *testing.T) {

diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go
@@ -428,9 +428,9 @@ func (t *HeadscaleInContainer) GetHostname() string {
 	return t.hostname
 }
 
-// WaitForReady blocks until the Headscale instance is ready to
+// WaitForRunning blocks until the Headscale instance is ready to
 // serve clients.
-func (t *HeadscaleInContainer) WaitForReady() error {
+func (t *HeadscaleInContainer) WaitForRunning() error {
 	url := t.GetHealthEndpoint()
 
 	log.Printf("waiting for headscale to be ready at %s", url)