Remove sourcemap/test and elasticsearch/estest

beater/beater_test.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -40,7 +41,6 @@ import (
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/apm-server/elasticsearch"
 	"github.com/elastic/apm-server/model"
-	"github.com/elastic/apm-server/sourcemap/test"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/instrumentation"
@@ -268,11 +268,13 @@ func TestTransformConfigIndex(t *testing.T) {
 	t.Run("with-observer-version", func(t *testing.T) { test(t, "blah-%{[observer.version]}-blah", "blah-1.2.3-blah") })
 }
 
+var validSourcemap, _ = ioutil.ReadFile("../testdata/sourcemap/bundle.js.map")
+
 func TestStoreUsesRUMElasticsearchConfig(t *testing.T) {
 	var called bool
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		called = true
-		w.Write([]byte(test.ValidSourcemap))
+		w.Write(validSourcemap)
 	}))
 	defer ts.Close()
 
@@ -298,7 +300,7 @@ func TestFleetStoreUsed(t *testing.T) {
 		called = true
 		wr := zlib.NewWriter(w)
 		defer wr.Close()
-		wr.Write([]byte(fmt.Sprintf(`{"sourceMap":%s}`, test.ValidSourcemap)))
+		wr.Write([]byte(fmt.Sprintf(`{"sourceMap":%s}`, validSourcemap)))
 	}))
 	defer ts.Close()
 

sourcemap/es_store_test.go

@@ -18,55 +18,60 @@
 package sourcemap
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"io"
 	"net/http"
+	"net/http/httptest"
 	"testing"
+	"time"
 
 	"github.com/go-sourcemap/sourcemap"
-
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
 
 	"github.com/elastic/apm-server/elasticsearch"
-
-	"github.com/elastic/apm-server/elasticsearch/estest"
 	logs "github.com/elastic/apm-server/log"
-	"github.com/elastic/apm-server/sourcemap/test"
 )
 
 func Test_esFetcher_fetchError(t *testing.T) {
 	for name, tc := range map[string]struct {
-		statusCode int
-		esBody     map[string]interface{}
-		temporary  bool
+		statusCode   int
+		clientError  bool
+		responseBody io.Reader
+		temporary    bool
 	}{
 		"es not reachable": {
-			statusCode: -1, temporary: true,
+			clientError: true,
+			temporary:   true,
 		},
 		"es bad request": {
 			statusCode: http.StatusBadRequest,
 		},
 		"empty sourcemap string": {
-			esBody: map[string]interface{}{
-				"hits": map[string]interface{}{
-					"total": map[string]interface{}{"value": 1},
-					"hits": []map[string]interface{}{
-						{"_source": map[string]interface{}{
-							"sourcemap": map[string]interface{}{
-								"sourcemap": ""}}}}}},
+			statusCode: http.StatusOK,
+			responseBody: sourcemapSearchResponseBody(1, []map[string]interface{}{{
+				"_source": map[string]interface{}{
+					"sourcemap": map[string]interface{}{
+						"sourcemap": "",
+					},
+				},
+			}}),
 		},
 	} {
 		t.Run(name, func(t *testing.T) {
-			statusCode := tc.statusCode
-			if statusCode == 0 {
-				statusCode = http.StatusOK
+			var client elasticsearch.Client
+			if tc.clientError {
+				client = newUnavailableElasticsearchClient(t)
+			} else {
+				client = newMockElasticsearchClient(t, tc.statusCode, tc.responseBody)
 			}
-			client, err := estest.NewElasticsearchClient(estest.NewTransport(t, statusCode, tc.esBody))
-			require.NoError(t, err)
+
 			consumer, err := testESStore(client).fetch(context.Background(), "abc", "1.0", "/tmp")
-			require.Error(t, err)
 			if tc.temporary {
 				assert.Contains(t, err.Error(), errMsgESFailure)
 			} else {
@@ -79,15 +84,27 @@ func Test_esFetcher_fetchError(t *testing.T) {
 
 func Test_esFetcher_fetch(t *testing.T) {
 	for name, tc := range map[string]struct {
-		client   elasticsearch.Client
-		filePath string
+		statusCode   int
+		responseBody io.Reader
+		filePath     string
 	}{
-		"no sourcemap found":                {client: test.ESClientWithSourcemapNotFound(t)},
-		"sourcemap indicated but not found": {client: test.ESClientWithSourcemapIndicatedNotFound(t)},
-		"valid sourcemap found":             {client: test.ESClientWithValidSourcemap(t), filePath: "bundle.js"},
+		"no sourcemap found": {
+			statusCode:   http.StatusNotFound,
+			responseBody: sourcemapSearchResponseBody(0, nil),
+		},
+		"sourcemap indicated but not found": {
+			statusCode:   http.StatusOK,
+			responseBody: sourcemapSearchResponseBody(1, []map[string]interface{}{}),
+		},
+		"valid sourcemap found": {
+			statusCode:   http.StatusOK,
+			responseBody: sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(validSourcemap)}),
+			filePath:     "bundle.js",
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
-			sourcemapStr, err := testESStore(tc.client).fetch(context.Background(), "abc", "1.0", "/tmp")
+			client := newMockElasticsearchClient(t, tc.statusCode, tc.responseBody)
+			sourcemapStr, err := testESStore(client).fetch(context.Background(), "abc", "1.0", "/tmp")
 			require.NoError(t, err)
 
 			if tc.filePath == "" {
@@ -104,3 +121,101 @@ func Test_esFetcher_fetch(t *testing.T) {
 func testESStore(client elasticsearch.Client) *esStore {
 	return &esStore{client: client, index: "apm-sourcemap", logger: logp.NewLogger(logs.Sourcemap)}
 }
+
+func sourcemapSearchResponseBody(hitsTotal int, hits []map[string]interface{}) io.Reader {
+	resultHits := map[string]interface{}{
+		"total": map[string]interface{}{
+			"value": hitsTotal,
+		},
+	}
+	if hits != nil {
+		resultHits["hits"] = hits
+	}
+	result := map[string]interface{}{"hits": resultHits}
+	data, err := json.Marshal(result)
+	if err != nil {
+		panic(err)
+	}
+	return bytes.NewReader(data)
+}
+
+func sourcemapHit(sourcemap string) map[string]interface{} {
+	return map[string]interface{}{
+		"_source": map[string]interface{}{
+			"sourcemap": map[string]interface{}{
+				"sourcemap": sourcemap,
+			},
+		},
+	}
+}
+
+// newUnavailableElasticsearchClient returns an elasticsearch.Client configured
+// to send requests to an invalid (unavailable) host.
+func newUnavailableElasticsearchClient(t testing.TB) elasticsearch.Client {
+	var transport roundTripperFunc = func(r *http.Request) (*http.Response, error) {
+		return nil, errors.New("client error")
+	}
+	backoff := func(int) time.Duration { return 0 }
+	client, err := elasticsearch.NewVersionedClient("", "", "", []string{"testing.invalid"}, nil, transport, 1, backoff)
+	require.NoError(t, err)
+	return client
+}
+
+// newMockElasticsearchClient returns an elasticsearch.Clien configured to send
+// requests to an httptest.Server that responds to source map search requests
+// with the given status code and response body.
+func newMockElasticsearchClient(t testing.TB, statusCode int, responseBody io.Reader) elasticsearch.Client {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(statusCode)
+		if responseBody != nil {
+			io.Copy(w, responseBody)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	config := elasticsearch.DefaultConfig()
+	config.Backoff.Init = time.Nanosecond
+	config.Hosts = []string{srv.URL}
+	client, err := elasticsearch.NewClient(config)
+	require.NoError(t, err)
+	return client
+}
+
+// validSourcemap is an example of a valid sourcemap for use in tests.
+const validSourcemap = `{
+    "version": 3,
+    "sources": [
+        "webpack:///bundle.js",
+        "",
+        "webpack:///./scripts/index.js",
+        "webpack:///./index.html",
+        "webpack:///./scripts/app.js"
+    ],
+    "names": [
+        "modules",
+        "__webpack_require__",
+        "moduleId",
+        "installedModules",
+        "exports",
+        "module",
+        "id",
+        "loaded",
+        "call",
+        "m",
+        "c",
+        "p",
+        "foo",
+        "console",
+        "log",
+        "foobar"
+    ],
+    "mappings": "CAAS,SAAUA,GCInB,QAAAC,GAAAC,GAGA,GAAAC,EAAAD,GACA,MAAAC,GAAAD,GAAAE,OAGA,IAAAC,GAAAF,EAAAD,IACAE,WACAE,GAAAJ,EACAK,QAAA,EAUA,OANAP,GAAAE,GAAAM,KAAAH,EAAAD,QAAAC,IAAAD,QAAAH,GAGAI,EAAAE,QAAA,EAGAF,EAAAD,QAvBA,GAAAD,KAqCA,OATAF,GAAAQ,EAAAT,EAGAC,EAAAS,EAAAP,EAGAF,EAAAU,EAAA,GAGAV,EAAA,KDMM,SAASI,EAAQD,EAASH,GE3ChCA,EAAA,GAEAA,EAAA,GAEAW,OFmDM,SAASP,EAAQD,EAASH,GGxDhCI,EAAAD,QAAAH,EAAAU,EAAA,cH8DM,SAASN,EAAQD,GI9DvB,QAAAQ,KACAC,QAAAC,IAAAC,QAGAH",
+    "file": "bundle.js",
+    "sourcesContent": [
+        "/******/ (function(modules) { // webpackBootstrap\n/******/ \t// The module cache\n/******/ \tvar installedModules = {};\n/******/\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(installedModules[moduleId])\n/******/ \t\t\treturn installedModules[moduleId].exports;\n/******/\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = installedModules[moduleId] = {\n/******/ \t\t\texports: {},\n/******/ \t\t\tid: moduleId,\n/******/ \t\t\tloaded: false\n/******/ \t\t};\n/******/\n/******/ \t\t// Execute the module function\n/******/ \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n/******/\n/******/ \t\t// Flag the module as loaded\n/******/ \t\tmodule.loaded = true;\n/******/\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/\n/******/\n/******/ \t// expose the modules object (__webpack_modules__)\n/******/ \t__webpack_require__.m = modules;\n/******/\n/******/ \t// expose the module cache\n/******/ \t__webpack_require__.c = installedModules;\n/******/\n/******/ \t// __webpack_public_path__\n/******/ \t__webpack_require__.p = \"\";\n/******/\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(0);\n/******/ })\n/************************************************************************/\n/******/ ([\n/* 0 */\n/***/ function(module, exports, __webpack_require__) {\n\n\t// Webpack\n\t__webpack_require__(1)\n\t\n\t__webpack_require__(2)\n\t\n\tfoo()\n\n\n/***/ },\n/* 1 */\n/***/ function(module, exports, __webpack_require__) {\n\n\tmodule.exports = __webpack_require__.p + \"index.html\"\n\n/***/ },\n/* 2 */\n/***/ function(module, exports) {\n\n\tfunction foo() {\n\t    console.log(foobar)\n\t}\n\t\n\tfoo()\n\n\n/***/ }\n/******/ ]);\n\n\n/** WEBPACK FOOTER **\n ** bundle.js\n **/",
+        " \t// The module cache\n \tvar installedModules = {};\n\n \t// The require function\n \tfunction __webpack_require__(moduleId) {\n\n \t\t// Check if module is in cache\n \t\tif(installedModules[moduleId])\n \t\t\treturn installedModules[moduleId].exports;\n\n \t\t// Create a new module (and put it into the cache)\n \t\tvar module = installedModules[moduleId] = {\n \t\t\texports: {},\n \t\t\tid: moduleId,\n \t\t\tloaded: false\n \t\t};\n\n \t\t// Execute the module function\n \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n\n \t\t// Flag the module as loaded\n \t\tmodule.loaded = true;\n\n \t\t// Return the exports of the module\n \t\treturn module.exports;\n \t}\n\n\n \t// expose the modules object (__webpack_modules__)\n \t__webpack_require__.m = modules;\n\n \t// expose the module cache\n \t__webpack_require__.c = installedModules;\n\n \t// __webpack_public_path__\n \t__webpack_require__.p = \"\";\n\n \t// Load entry module and return exports\n \treturn __webpack_require__(0);\n\n\n\n/** WEBPACK FOOTER **\n ** webpack/bootstrap 6002740481c9666b0d38\n **/",
+        "// Webpack\nrequire('../index.html')\n\nrequire('./app')\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/index.js\n ** module id = 0\n ** module chunks = 0\n **/",
+        "module.exports = __webpack_public_path__ + \"index.html\"\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./index.html\n ** module id = 1\n ** module chunks = 0\n **/",
+        "function foo() {\n    console.log(foobar)\n}\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/app.js\n ** module id = 2\n ** module chunks = 0\n **/"
+    ],
+    "sourceRoot": ""
+}`

sourcemap/mapper_test.go

@@ -24,8 +24,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/elastic/apm-server/sourcemap/test"
-
 	"github.com/go-sourcemap/sourcemap"
 	"github.com/stretchr/testify/assert"
 )
@@ -37,7 +35,7 @@ func TestMapNilConsumer(t *testing.T) {
 }
 
 func TestMapNoMatch(t *testing.T) {
-	m, err := sourcemap.Parse("", []byte(test.ValidSourcemap))
+	m, err := sourcemap.Parse("", []byte(validSourcemap))
 	require.NoError(t, err)
 
 	// nothing found for lineno and colno
@@ -51,8 +49,6 @@ func TestMapNoMatch(t *testing.T) {
 }
 
 func TestMapMatch(t *testing.T) {
-	validSourcemap := test.ValidSourcemap
-
 	// Re-encode the sourcemap, adding carriage returns to the
 	// line endings in the source content.
 	decoded := make(map[string]interface{})