[Heartbeat] Add httpcommon options to ZipURL (#27699) (#27790)

heartbeat/docs/monitors/monitor-browser.asciidoc

@@ -65,6 +65,7 @@ Under `zip_url`, specify these options:
 located in the repository.
 *`username`*:: The username for authenticating with the zip endpoint. This setting is optional.
 *`password`*:: The password for authenticating with the zip endpoint. This setting is optional.
+*`ssl`*:: SSL options applied to downloading the zip, not the browser. See <<configuration-ssl>> for more details.
 
 If `username` and `password` are provided, they will be sent as HTTP Basic Authentication
 headers to the remote zip endpoint.
@@ -83,9 +84,11 @@ Example configuration:
       folder: "examples/todos"
       username: ""
       password: ""
+      # ssl options apply to downloading the zip, not the browser
+      #ssl:
+      #  certificate_authorities: ['/etc/ca.crt']
 -------------------------------------------------------------------------------
 
-
 [float]
 [[monitor-source-local]]
 ===== `Local directory`
@@ -198,7 +201,6 @@ Example configuration:
 *`tags`*:: run only journeys with the given tag(s), or globs
 *`match`*:: run only journeys with a name or tags that matches the configured glob
 
-
 [float]
 [[monitor-browser-synthetics-args]]
 ==== `synthetics_args`

monitors.d/plaintodos.yml

@@ -0,0 +1,12 @@
+- name: Todos
+  id: todos
+  type: browser
+  enabled: true
+  schedule: "@every 3m"
+  tags: todos-app
+  params:
+    url: "https://elastic.github.io/synthetics-demo/"
+  source:
+    zip_url: 
+      url: "https://github.com/elastic/synthetics-demo/archive/refs/heads/main.zip"
+      folder: "todos/synthetics-tests"

x-pack/heartbeat/monitors/browser/source/zipurl.go

@@ -14,6 +14,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/elastic/beats/v7/libbeat/common/transport/httpcommon"
 )
 
 type ZipURLSource struct {
@@ -23,13 +25,25 @@ type ZipURLSource struct {
 	Password string `config:"password" json:"password"`
 	Retries  int    `config:"retries" default:"3" json:"retries"`
 	BaseSource
-	// Etag from last successful fetch
-	etag            string
 	TargetDirectory string `config:"target_directory" json:"target_directory"`
+
+	// Etag from last successful fetch
+	etag string
+
+	Transport httpcommon.HTTPTransportSettings `config:",inline" yaml:",inline"`
+
+	httpClient *http.Client
 }
 
 var ErrNoEtag = fmt.Errorf("No ETag header in zip file response. Heartbeat requires an etag to efficiently cache downloaded code")
 
+func (z *ZipURLSource) Validate() (err error) {
+	if z.httpClient == nil {
+		z.httpClient, _ = z.Transport.Client()
+	}
+	return err
+}
+
 func (z *ZipURLSource) Fetch() error {
 	changed, err := checkIfChanged(z)
 	if err != nil {
@@ -181,14 +195,15 @@ func retryingZipRequest(method string, z *ZipURLSource) (resp *http.Response, er
 }
 
 func zipRequest(method string, z *ZipURLSource) (*http.Response, error) {
+
 	req, err := http.NewRequest(method, z.URL, nil)
 	if err != nil {
 		return nil, fmt.Errorf("could not issue request to: %s %w", z.URL, err)
 	}
 	if z.Username != "" && z.Password != "" {
 		req.SetBasicAuth(z.Username, z.Password)
 	}
-	return http.DefaultClient.Do(req)
+	return z.httpClient.Do(req)
 }
 
 func download(z *ZipURLSource, tf *os.File) (etag string, err error) {

x-pack/heartbeat/monitors/browser/source/zipurl_test.go

@@ -5,69 +5,131 @@
 package source
 
 import (
-	"context"
 	"fmt"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source/fixtures"
 )
 
-func TestZipUrlFetchNoAuth(t *testing.T) {
-	address, teardown := setupTests()
-	defer teardown()
-
-	zus := &ZipURLSource{
-		URL:     fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:  "/",
-		Retries: 3,
+func TestSimpleCases(t *testing.T) {
+	type testCase struct {
+		name         string
+		cfg          common.MapStr
+		tlsServer    bool
+		wantFetchErr bool
+	}
+	testCases := []testCase{
+		{
+			"basics",
+			common.MapStr{
+				"folder":  "/",
+				"retries": 3,
+			},
+			false,
+			false,
+		},
+		{
+			"targetdir",
+			common.MapStr{
+				"folder":           "/",
+				"retries":          3,
+				"target_directory": "/tmp/synthetics/blah",
+			},
+			false,
+			false,
+		},
+		{
+			"auth success",
+			common.MapStr{
+				"folder":   "/",
+				"retries":  3,
+				"username": "testuser",
+				"password": "testpass",
+			},
+			false,
+			false,
+		},
+		{
+			"auth failure",
+			common.MapStr{
+				"folder":   "/",
+				"retries":  3,
+				"username": "testuser",
+				"password": "badpass",
+			},
+			false,
+			true,
+		},
+		{
+			"ssl ignore cert errors",
+			common.MapStr{
+				"folder":  "/",
+				"retries": 3,
+				"ssl": common.MapStr{
+					"enabled":           "true",
+					"verification_mode": "none",
+				},
+			},
+			true,
+			false,
+		},
+		{
+			"bad ssl",
+			common.MapStr{
+				"folder":  "/",
+				"retries": 3,
+				"ssl": common.MapStr{
+					"enabled":                 "true",
+					"certificate_authorities": []string{},
+				},
+			},
+			true,
+			true,
+		},
 	}
-	fetchAndCheckDir(t, zus)
-}
 
-func TestZipUrlFetchWithAuth(t *testing.T) {
-	address, teardown := setupTests()
-	defer teardown()
+	for _, tc := range testCases {
+		url, teardown := setupTests(tc.tlsServer)
+		defer teardown()
+		t.Run(tc.name, func(t *testing.T) {
+			tc.cfg["url"] = fmt.Sprintf("%s/fixtures/todos.zip", url)
+			zus, err := dummyZus(tc.cfg)
+			require.NoError(t, err)
 
-	zus := &ZipURLSource{
-		URL:      fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:   "/",
-		Retries:  3,
-		Username: "testuser",
-		Password: "testpass",
-	}
-	fetchAndCheckDir(t, zus)
-}
+			require.NotNil(t, zus.httpClient)
 
-func TestZipUrlTargetDirectory(t *testing.T) {
-	address, teardown := setupTests()
-	defer teardown()
+			if tc.wantFetchErr == true {
+				err := zus.Fetch()
+				require.Error(t, err)
+				return
+			}
 
-	zus := &ZipURLSource{
-		URL:             fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:          "/",
-		Retries:         3,
-		TargetDirectory: "/tmp/synthetics/blah",
+			fetchAndCheckDir(t, zus)
+		})
 	}
-	fetchAndCheckDir(t, zus)
 }
 
 func TestZipUrlWithSameEtag(t *testing.T) {
-	address, teardown := setupTests()
+	address, teardown := setupTests(false)
 	defer teardown()
 
-	zus := ZipURLSource{
-		URL:     fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:  "/",
-		Retries: 3,
-	}
-	err := zus.Fetch()
+	zus, err := dummyZus(common.MapStr{
+		"url":     fmt.Sprintf("%s/fixtures/todos.zip", address),
+		"folder":  "/",
+		"retries": 3,
+	})
+	require.NoError(t, err)
+	err = zus.Fetch()
 	defer zus.Close()
 	require.NoError(t, err)
 
@@ -80,32 +142,33 @@ func TestZipUrlWithSameEtag(t *testing.T) {
 }
 
 func TestZipUrlWithBadUrl(t *testing.T) {
-	_, teardown := setupTests()
+	_, teardown := setupTests(false)
 	defer teardown()
 
-	zus := ZipURLSource{
-		URL:     "http://notahost.notadomaintoehutoeuhn",
-		Folder:  "/",
-		Retries: 2,
-	}
-	err := zus.Fetch()
+	zus, err := dummyZus(common.MapStr{
+		"url":     "http://notahost.notadomaintoehutoeuhn",
+		"folder":  "/",
+		"retries": 2,
+	})
+	require.NoError(t, err)
+	err = zus.Fetch()
 	defer zus.Close()
 	require.Error(t, err)
 }
 
-func setupTests() (addr string, teardown func()) {
+func setupTests(tls bool) (addr string, teardown func()) {
 	// go offline, so we dont invoke npm install for unit tests
 	GoOffline()
 
-	srv := createServer()
-	address := srv.Addr
+	srv := createServer(tls)
+	address := srv.URL
 	return address, func() {
 		GoOnline()
-		srv.Shutdown(context.Background())
+		srv.Close()
 	}
 }
 
-func createServer() (addr *http.Server) {
+func createServer(tls bool) (addr *httptest.Server) {
 	_, filename, _, _ := runtime.Caller(0)
 	fixturesPath := path.Join(filepath.Dir(filename), "fixtures")
 	fileServer := http.FileServer(http.Dir(fixturesPath))
@@ -121,10 +184,12 @@ func createServer() (addr *http.Server) {
 		http.StripPrefix("/fixtures", fileServer).ServeHTTP(resp, req)
 	})
 
-	srv := &http.Server{Addr: "localhost:1234", Handler: mux}
-	go func() {
-		srv.ListenAndServe()
-	}()
+	var srv *httptest.Server
+	if tls {
+		srv = httptest.NewTLSServer(mux)
+	} else {
+		srv = httptest.NewServer(mux)
+	}
 
 	return srv
 }
@@ -140,3 +205,14 @@ func fetchAndCheckDir(t *testing.T, zip *ZipURLSource) {
 	_, err = os.Stat(zip.TargetDirectory)
 	require.True(t, os.IsNotExist(err), "TargetDirectory %s should have been deleted", zip.TargetDirectory)
 }
+
+func dummyZus(conf map[string]interface{}) (*ZipURLSource, error) {
+	zus := &ZipURLSource{}
+	y, _ := yaml.Marshal(conf)
+	c, err := common.NewConfigWithYAML(y, string(y))
+	if err != nil {
+		return nil, err
+	}
+	err = c.Unpack(zus)
+	return zus, err
+}