From 1870bb116d6a97341c36b9aaea1ac29c507984de Mon Sep 17 00:00:00 2001 From: codebien <2103732+codebien@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:32:15 +0200 Subject: [PATCH] Drop the deprecated convert command --- cmd/convert.go | 133 --------- cmd/convert_test.go | 169 ------------ cmd/root.go | 2 +- cmd/tests/cmd_run_test.go | 4 - converter/har/converter.go | 470 -------------------------------- converter/har/converter_test.go | 91 ------- converter/har/types.go | 234 ---------------- converter/har/utils.go | 85 ------ converter/har/utils_test.go | 68 ----- go.mod | 4 +- 10 files changed, 3 insertions(+), 1257 deletions(-) delete mode 100644 cmd/convert.go delete mode 100644 cmd/convert_test.go delete mode 100644 converter/har/converter.go delete mode 100644 converter/har/converter_test.go delete mode 100644 converter/har/types.go delete mode 100644 converter/har/utils.go delete mode 100644 converter/har/utils_test.go diff --git a/cmd/convert.go b/cmd/convert.go deleted file mode 100644 index e3fc2ce6bea8..000000000000 --- a/cmd/convert.go +++ /dev/null @@ -1,133 +0,0 @@ -package cmd - -import ( - "encoding/json" - "io" - - "github.com/spf13/cobra" - "gopkg.in/guregu/null.v3" - - "go.k6.io/k6/cmd/state" - "go.k6.io/k6/converter/har" - "go.k6.io/k6/lib" - "go.k6.io/k6/lib/fsext" -) - -// TODO: split apart like `k6 run` and `k6 archive`? -// -//nolint:funlen,gocognit -func getCmdConvert(gs *state.GlobalState) *cobra.Command { - var ( - convertOutput string - optionsFilePath string - minSleep uint - maxSleep uint - enableChecks bool - returnOnFailedCheck bool - correlate bool - threshold uint - nobatch bool - only []string - skip []string - ) - - exampleText := getExampleText(gs, ` - # Convert a HAR file to a k6 script. - {{.}} convert -O har-session.js session.har - - # Convert a HAR file to a k6 script creating requests only for the given domain/s. - {{.}} convert -O har-session.js --only yourdomain.com,additionaldomain.com session.har - - # Convert a HAR file. Batching requests together as long as idle time between requests <800ms - {{.}} convert --batch-threshold 800 session.har - - # Run the k6 script. - {{.}} run har-session.js`[1:]) - - convertCmd := &cobra.Command{ - Use: "convert", - Short: "Convert a HAR file to a k6 script", - Long: "Convert a HAR (HTTP Archive) file to a k6 script", - Deprecated: "please use har-to-k6 (https://github.com/grafana/har-to-k6) instead.", - Example: exampleText, - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - // Parse the HAR file - r, err := gs.FS.Open(args[0]) - if err != nil { - return err - } - h, err := har.Decode(r) - if err != nil { - return err - } - if err = r.Close(); err != nil { - return err - } - - // recordings include redirections as separate requests, and we dont want to trigger them twice - options := lib.Options{MaxRedirects: null.IntFrom(0)} - - if optionsFilePath != "" { - optionsFileContents, readErr := fsext.ReadFile(gs.FS, optionsFilePath) - if readErr != nil { - return readErr - } - var injectedOptions lib.Options - if err := json.Unmarshal(optionsFileContents, &injectedOptions); err != nil { - return err - } - options = options.Apply(injectedOptions) - } - - // TODO: refactor... - script, err := har.Convert(h, options, minSleep, maxSleep, enableChecks, - returnOnFailedCheck, threshold, nobatch, correlate, only, skip) - if err != nil { - return err - } - - // Write script content to stdout or file - if convertOutput == "" || convertOutput == "-" { //nolint:nestif - if _, err := io.WriteString(gs.Stdout, script); err != nil { - return err - } - } else { - f, err := gs.FS.Create(convertOutput) - if err != nil { - return err - } - if _, err := f.WriteString(script); err != nil { - return err - } - if err := f.Sync(); err != nil { - return err - } - if err := f.Close(); err != nil { - return err - } - } - return nil - }, - } - - convertCmd.Flags().SortFlags = false - convertCmd.Flags().StringVarP( - &convertOutput, "output", "O", convertOutput, - "k6 script output filename (stdout by default)", - ) - convertCmd.Flags().StringVarP( - &optionsFilePath, "options", "", optionsFilePath, - "path to a JSON file with options that would be injected in the output script", - ) - convertCmd.Flags().StringSliceVarP(&only, "only", "", []string{}, "include only requests from the given domains") - convertCmd.Flags().StringSliceVarP(&skip, "skip", "", []string{}, "skip requests from the given domains") - convertCmd.Flags().UintVarP(&threshold, "batch-threshold", "", 500, "batch request idle time threshold (see example)") - convertCmd.Flags().BoolVarP(&nobatch, "no-batch", "", false, "don't generate batch calls") - convertCmd.Flags().BoolVarP(&enableChecks, "enable-status-code-checks", "", false, "add a status code check for each HTTP response") //nolint:lll - convertCmd.Flags().BoolVarP(&returnOnFailedCheck, "return-on-failed-check", "", false, "return from iteration if we get an unexpected response status code") //nolint:lll - convertCmd.Flags().BoolVarP(&correlate, "correlate", "", false, "detect values in responses being used in subsequent requests and try adapt the script accordingly (only redirects and JSON values for now)") //nolint:lll - convertCmd.Flags().UintVarP(&minSleep, "min-sleep", "", 20, "the minimum amount of seconds to sleep after each iteration") //nolint:lll - convertCmd.Flags().UintVarP(&maxSleep, "max-sleep", "", 40, "the maximum amount of seconds to sleep after each iteration") //nolint:lll - return convertCmd -} diff --git a/cmd/convert_test.go b/cmd/convert_test.go deleted file mode 100644 index d20535d99b78..000000000000 --- a/cmd/convert_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package cmd - -import ( - "os" - "regexp" - "testing" - - "github.com/pmezard/go-difflib/difflib" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.k6.io/k6/cmd/tests" - "go.k6.io/k6/lib/fsext" -) - -const testHAR = ` -{ - "log": { - "version": "1.2", - "creator": { - "name": "WebInspector", - "version": "537.36" - }, - "pages": [ - { - "startedDateTime": "2018-01-21T19:48:40.432Z", - "id": "page_2", - "title": "https://golang.org/", - "pageTimings": { - "onContentLoad": 590.3389999875799, - "onLoad": 1593.1009999476373 - } - } - ], - "entries": [ - { - "startedDateTime": "2018-01-21T19:48:40.587Z", - "time": 147.5899999756366, - "request": { - "method": "GET", - "url": "https://golang.org/", - "httpVersion": "http/2.0+quic/39", - "headers": [ - { - "name": "pragma", - "value": "no-cache" - } - ], - "queryString": [], - "cookies": [], - "headersSize": -1, - "bodySize": 0 - }, - "cache": {}, - "timings": { - "blocked": 0.43399997614324004, - "dns": -1, - "ssl": -1, - "connect": -1, - "send": 0.12700003571808005, - "wait": 149.02899996377528, - "receive": 0, - "_blocked_queueing": -1 - }, - "serverIPAddress": "172.217.22.177", - "pageref": "page_2" - } - ] - } -} -` - -const testHARConvertResult = `import { group, sleep } from 'k6'; -import http from 'k6/http'; - -// Version: 1.2 -// Creator: WebInspector - -export let options = { - maxRedirects: 0, -}; - -export default function() { - - group("page_2 - https://golang.org/", function() { - let req, res; - req = [{ - "method": "get", - "url": "https://golang.org/", - "params": { - "headers": { - "pragma": "no-cache" - } - } - }]; - res = http.batch(req); - // Random sleep between 20s and 40s - sleep(Math.floor(Math.random()*20+20)); - }); - -} -` - -func TestConvertCmdCorrelate(t *testing.T) { - t.Parallel() - har, err := os.ReadFile("testdata/example.har") //nolint:forbidigo - require.NoError(t, err) - - expectedTestPlan, err := os.ReadFile("testdata/example.js") //nolint:forbidigo - require.NoError(t, err) - - ts := tests.NewGlobalTestState(t) - require.NoError(t, fsext.WriteFile(ts.FS, "correlate.har", har, 0o644)) - ts.CmdArgs = []string{ - "k6", "convert", "--output=result.js", "--correlate=true", "--no-batch=true", - "--enable-status-code-checks=true", "--return-on-failed-check=true", "correlate.har", - } - - newRootCommand(ts.GlobalState).execute() - - result, err := fsext.ReadFile(ts.FS, "result.js") - require.NoError(t, err) - - // Sanitizing to avoid windows problems with carriage returns - re := regexp.MustCompile(`\r`) - expected := re.ReplaceAllString(string(expectedTestPlan), ``) - resultStr := re.ReplaceAllString(string(result), ``) - - if assert.NoError(t, err) { - // assert.Equal suppresses the diff it is too big, so we add it as the test error message manually as well. - diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ - A: difflib.SplitLines(expected), - B: difflib.SplitLines(resultStr), - FromFile: "Expected", - FromDate: "", - ToFile: "Actual", - ToDate: "", - Context: 1, - }) - - assert.Equal(t, expected, resultStr, diff) - } -} - -func TestConvertCmdStdout(t *testing.T) { - t.Parallel() - ts := tests.NewGlobalTestState(t) - require.NoError(t, fsext.WriteFile(ts.FS, "stdout.har", []byte(testHAR), 0o644)) - ts.CmdArgs = []string{"k6", "convert", "stdout.har"} - - newRootCommand(ts.GlobalState).execute() - assert.Equal(t, "Command \"convert\" is deprecated, please use har-to-k6 (https://github.com/grafana/har-to-k6) instead.\n"+testHARConvertResult, ts.Stdout.String()) -} - -func TestConvertCmdOutputFile(t *testing.T) { - t.Parallel() - - ts := tests.NewGlobalTestState(t) - require.NoError(t, fsext.WriteFile(ts.FS, "output.har", []byte(testHAR), 0o644)) - ts.CmdArgs = []string{"k6", "convert", "--output", "result.js", "output.har"} - - newRootCommand(ts.GlobalState).execute() - - output, err := fsext.ReadFile(ts.FS, "result.js") - assert.NoError(t, err) - assert.Equal(t, testHARConvertResult, string(output)) -} - -// TODO: test options injection; right now that's difficult because when there are multiple -// options, they can be emitted in different order in the JSON diff --git a/cmd/root.go b/cmd/root.go index 9de1d625c98f..a16f558da126 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -57,7 +57,7 @@ func newRootCommand(gs *state.GlobalState) *rootCommand { rootCmd.SetIn(gs.Stdin) subCommands := []func(*state.GlobalState) *cobra.Command{ - getCmdArchive, getCmdCloud, getCmdConvert, getCmdInspect, + getCmdArchive, getCmdCloud, getCmdInspect, getCmdLogin, getCmdPause, getCmdResume, getCmdScale, getCmdRun, getCmdStats, getCmdStatus, getCmdVersion, } diff --git a/cmd/tests/cmd_run_test.go b/cmd/tests/cmd_run_test.go index ce8fc6cabad7..cd2d57d68a0f 100644 --- a/cmd/tests/cmd_run_test.go +++ b/cmd/tests/cmd_run_test.go @@ -99,10 +99,6 @@ func TestBinaryNameHelpStdout(t *testing.T) { cmdName: "cloud", containsOutput: fmt.Sprintf("%s cloud script.js", ts.BinaryName), }, - { - cmdName: "convert", - containsOutput: fmt.Sprintf("%s convert -O har-session.js session.har", ts.BinaryName), - }, { cmdName: "login", extraCmd: "cloud", diff --git a/converter/har/converter.go b/converter/har/converter.go deleted file mode 100644 index 7391a93afa90..000000000000 --- a/converter/har/converter.go +++ /dev/null @@ -1,470 +0,0 @@ -package har - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "sort" - "strings" - - "github.com/tidwall/pretty" - - "go.k6.io/k6/lib" -) - -// fprint panics when where's an error writing to the supplied io.Writer -// since this will be used on in-memory expandable buffers, that should -// happen only when we run out of memory... -func fprint(w io.Writer, a ...interface{}) int { - n, err := fmt.Fprint(w, a...) - if err != nil { - panic(err.Error()) - } - return n -} - -// fprintf panics when where's an error writing to the supplied io.Writer -// since this will be used on in-memory expandable buffers, that should -// happen only when we run out of memory... -func fprintf(w io.Writer, format string, a ...interface{}) int { - n, err := fmt.Fprintf(w, format, a...) - if err != nil { - panic(err.Error()) - } - return n -} - -// TODO: refactor this to have fewer parameters... or just refactor in general... -func Convert(h HAR, options lib.Options, minSleep, maxSleep uint, enableChecks bool, returnOnFailedCheck bool, batchTime uint, nobatch bool, correlate bool, only, skip []string) (result string, convertErr error) { - var b bytes.Buffer - w := bufio.NewWriter(&b) - - if returnOnFailedCheck && !enableChecks { - return "", fmt.Errorf("return on failed check requires --enable-status-code-checks") - } - - if correlate && !nobatch { - return "", fmt.Errorf("correlation requires --no-batch") - } - - if h.Log == nil { - return "", fmt.Errorf("invalid HAR file supplied, the 'log' property is missing") - } - - if enableChecks { - fprint(w, "import { group, check, sleep } from 'k6';\n") - } else { - fprint(w, "import { group, sleep } from 'k6';\n") - } - fprint(w, "import http from 'k6/http';\n\n") - - fprintf(w, "// Version: %v\n", h.Log.Version) - fprintf(w, "// Creator: %v\n", h.Log.Creator.Name) - if h.Log.Browser != nil { - fprintf(w, "// Browser: %v\n", h.Log.Browser.Name) - } - if h.Log.Comment != "" { - fprintf(w, "// %v\n", h.Log.Comment) - } - - fprint(w, "\nexport let options = {\n") - options.ForEachSpecified("json", func(key string, val interface{}) { - if valJSON, err := json.MarshalIndent(val, " ", " "); err != nil { - convertErr = err - } else { - fprintf(w, " %s: %s,\n", key, valJSON) - } - }) - if convertErr != nil { - return "", convertErr - } - fprint(w, "};\n\n") - - fprint(w, "export default function() {\n\n") - - pages := h.Log.Pages - sort.Sort(PageByStarted(pages)) - - // Hack to handle HAR files without a pages array - // Temporary fix for https://github.com/k6io/k6/issues/793 - if len(pages) == 0 { - pages = []Page{{ - ID: "", // The Pageref property of all Entries will be an empty string - Title: "Global", - Comment: "Placeholder page since there were no pages specified in the HAR file", - }} - } - - // Grouping by page and URL filtering - pageEntries := make(map[string][]*Entry) - for _, e := range h.Log.Entries { - - // URL filtering - u, err := url.Parse(e.Request.URL) - if err != nil { - return "", err - } - if !IsAllowedURL(u.Host, only, skip) { - continue - } - - // Avoid multipart/form-data requests until k6 scripts can support binary data - if e.Request.PostData != nil && strings.HasPrefix(e.Request.PostData.MimeType, "multipart/form-data") { - continue - } - - // Create new group o adding page to a existing one - if _, ok := pageEntries[e.Pageref]; !ok { - pageEntries[e.Pageref] = append([]*Entry{}, e) - } else { - pageEntries[e.Pageref] = append(pageEntries[e.Pageref], e) - } - } - - for i, page := range pages { - - entries := pageEntries[page.ID] - - scriptGroupName := page.ID + " - " + page.Title - if page.ID == "" { - // Temporary fix for https://github.com/k6io/k6/issues/793 - // I can't just remove the group() call since all of the subsequent code indentation is hardcoded... - scriptGroupName = page.Title - } - fprintf(w, "\tgroup(%q, function() {\n", scriptGroupName) - - sort.Sort(EntryByStarted(entries)) - - if nobatch { - var recordedRedirectURL string - previousResponse := map[string]interface{}{} - - fprint(w, "\t\tlet res, redirectUrl, json;\n") - - for entryIndex, e := range entries { - - var params []string - var cookies []string - var body string - - fprintf(w, "\t\t// Request #%d\n", entryIndex) - - if e.Request.PostData != nil { - body = e.Request.PostData.Text - } - - for _, c := range e.Request.Cookies { - cookies = append(cookies, fmt.Sprintf(`%q: %q`, c.Name, c.Value)) - } - if len(cookies) > 0 { - params = append(params, fmt.Sprintf("\"cookies\": {\n\t\t\t\t%s\n\t\t\t}", strings.Join(cookies, ",\n\t\t\t\t\t"))) - } - - if headers := buildK6Headers(e.Request.Headers); len(headers) > 0 { - params = append(params, fmt.Sprintf("\"headers\": {\n\t\t\t\t\t%s\n\t\t\t\t}", strings.Join(headers, ",\n\t\t\t\t\t"))) - } - - fprintf(w, "\t\tres = http.%s(", strings.ToLower(e.Request.Method)) - - if correlate && recordedRedirectURL != "" { - if recordedRedirectURL != e.Request.URL { - return "", errors.New( //nolint:stylecheck - "The har file contained a redirect but the next request did not match that redirect. " + - "Possibly a misbehaving client or concurrent requests?", - ) - } - fprintf(w, "redirectUrl") - recordedRedirectURL = "" - } else { - fprintf(w, "%q", e.Request.URL) - } - - if e.Request.Method != http.MethodGet { - if correlate && e.Request.PostData != nil && strings.Contains(e.Request.PostData.MimeType, "json") { - requestMap := map[string]interface{}{} - - escapedPostdata := strings.Replace(e.Request.PostData.Text, "$", "\\$", -1) - - if err := json.Unmarshal([]byte(escapedPostdata), &requestMap); err != nil { - return "", err - } - - if len(previousResponse) != 0 { - traverseMaps(requestMap, previousResponse, nil) - } - requestText, err := json.Marshal(requestMap) - if err == nil { - prettyJSONString := string(pretty.PrettyOptions(requestText, &pretty.Options{Width: 999999, Prefix: "\t\t\t", Indent: "\t", SortKeys: true})[:]) - fprintf(w, ",\n\t\t\t`%s`", strings.TrimSpace(prettyJSONString)) - } else { - return "", err - } - - } else { - fprintf(w, ",\n\t\t%q", body) - } - } - - if len(params) > 0 { - fprintf(w, ",\n\t\t\t{\n\t\t\t\t%s\n\t\t\t}", strings.Join(params, ",\n\t\t\t")) - } - - fprintf(w, "\n\t\t)\n") - - if e.Response != nil { - // the response is nil if there is a failed request in the recording, or if responses were not recorded - if enableChecks { - if e.Response.Status > 0 { - if returnOnFailedCheck { - fprintf(w, "\t\tif (!check(res, {\"status is %v\": (r) => r.status === %v })) { return };\n", e.Response.Status, e.Response.Status) - } else { - fprintf(w, "\t\tcheck(res, {\"status is %v\": (r) => r.status === %v });\n", e.Response.Status, e.Response.Status) - } - } - } - - for _, header := range e.Response.Headers { - if header.Name == "Location" { - fprintf(w, "\t\tredirectUrl = res.headers.Location;\n") - recordedRedirectURL = header.Value - break - } - } - - responseMimeType := e.Response.Content.MimeType - if correlate && - strings.Index(responseMimeType, "application/") == 0 && - strings.Index(responseMimeType, "json") == len(responseMimeType)-4 { - if err := json.Unmarshal([]byte(e.Response.Content.Text), &previousResponse); err != nil { - return "", err - } - fprint(w, "\t\tjson = JSON.parse(res.body);\n") - } - } - } - } else { - batches := SplitEntriesInBatches(entries, batchTime) - - fprint(w, "\t\tlet req, res;\n") - - for j, batchEntries := range batches { - - fprint(w, "\t\treq = [") - for k, e := range batchEntries { - r, err := buildK6RequestObject(e.Request) - if err != nil { - return "", err - } - fprintf(w, "%v", r) - if k != len(batchEntries)-1 { - fprint(w, ",") - } - } - fprint(w, "];\n") - fprint(w, "\t\tres = http.batch(req);\n") - - if enableChecks { - for k, e := range batchEntries { - if e.Response.Status > 0 { - if returnOnFailedCheck { - fprintf(w, "\t\tif (!check(res, {\"status is %v\": (r) => r.status === %v })) { return };\n", e.Response.Status, e.Response.Status) - } else { - fprintf(w, "\t\tcheck(res[%v], {\"status is %v\": (r) => r.status === %v });\n", k, e.Response.Status, e.Response.Status) - } - } - } - } - - if j != len(batches)-1 { - lastBatchEntry := batchEntries[len(batchEntries)-1] - firstBatchEntry := batches[j+1][0] - t := firstBatchEntry.StartedDateTime.Sub(lastBatchEntry.StartedDateTime).Seconds() - fprintf(w, "\t\tsleep(%.2f);\n", t) - } - } - - if i == len(pages)-1 { - // Last page; add random sleep time at the group completion - fprintf(w, "\t\t// Random sleep between %ds and %ds\n", minSleep, maxSleep) - fprintf(w, "\t\tsleep(Math.floor(Math.random()*%d+%d));\n", maxSleep-minSleep, minSleep) - } else { - // Add sleep time at the end of the group - nextPage := pages[i+1] - sleepTime := 0.5 - if len(entries) > 0 { - lastEntry := entries[len(entries)-1] - t := nextPage.StartedDateTime.Sub(lastEntry.StartedDateTime).Seconds() - if t >= 0.01 { - sleepTime = t - } - } - fprintf(w, "\t\tsleep(%.2f);\n", sleepTime) - } - } - - fprint(w, "\t});\n") - } - - fprint(w, "\n}\n") - if err := w.Flush(); err != nil { - return "", err - } - return b.String(), nil -} - -func buildK6RequestObject(req *Request) (string, error) { - var b bytes.Buffer - w := bufio.NewWriter(&b) - - fprint(w, "{\n") - - method := strings.ToLower(req.Method) - if method == "delete" { - method = "del" - } - fprintf(w, `"method": %q, "url": %q`, method, req.URL) - - if req.PostData != nil && method != "get" { - postParams, plainText, err := buildK6Body(req) - if err != nil { - return "", err - } else if len(postParams) > 0 { - fprintf(w, `, "body": { %s }`, strings.Join(postParams, ", ")) - } else if plainText != "" { - fprintf(w, `, "body": %q`, plainText) - } - } - - var params []string - var cookies []string - for _, c := range req.Cookies { - cookies = append(cookies, fmt.Sprintf(`%q: %q`, c.Name, c.Value)) - } - if len(cookies) > 0 { - params = append(params, fmt.Sprintf(`"cookies": { %s }`, strings.Join(cookies, ", "))) - } - - if headers := buildK6Headers(req.Headers); len(headers) > 0 { - params = append(params, fmt.Sprintf(`"headers": { %s }`, strings.Join(headers, ", "))) - } - - if len(params) > 0 { - fprintf(w, `, "params": { %s }`, strings.Join(params, ", ")) - } - - fprint(w, "}") - if err := w.Flush(); err != nil { - return "", err - } - - var buffer bytes.Buffer - err := json.Indent(&buffer, b.Bytes(), "\t\t", "\t") - if err != nil { - return "", err - } - return buffer.String(), nil -} - -func buildK6Headers(headers []Header) []string { - var h []string - if len(headers) > 0 { - ignored := map[string]bool{"cookie": true, "content-length": true} - for _, header := range headers { - name := strings.ToLower(header.Name) - _, isIgnored := ignored[name] - // Avoid SPDY's, duplicated or ignored headers - if !isIgnored && name[0] != ':' { - ignored[name] = true - h = append(h, fmt.Sprintf("%q: %q", header.Name, header.Value)) - } - } - } - return h -} - -func buildK6Body(req *Request) ([]string, string, error) { - var postParams []string - if req.PostData.MimeType == "application/x-www-form-urlencoded" && len(req.PostData.Params) > 0 { - for _, p := range req.PostData.Params { - n, err := url.QueryUnescape(p.Name) - if err != nil { - return postParams, "", err - } - v, err := url.QueryUnescape(p.Value) - if err != nil { - return postParams, "", err - } - postParams = append(postParams, fmt.Sprintf(`%q: %q`, n, v)) - } - return postParams, "", nil - } - return postParams, req.PostData.Text, nil -} - -func traverseMaps(request map[string]interface{}, response map[string]interface{}, path []interface{}) { - if response == nil { - // previous call reached a leaf in the response map so there's no point continuing - return - } - for key, val := range request { - responseVal := response[key] - if responseVal == nil { - // no corresponding value in response map (and the type conversion below would fail so we need an early exit) - continue - } - newPath := append(path, key) - switch concreteVal := val.(type) { - case map[string]interface{}: - traverseMaps(concreteVal, responseVal.(map[string]interface{}), newPath) - case []interface{}: - traverseArrays(concreteVal, responseVal.([]interface{}), newPath) - default: - if responseVal == val { - request[key] = jsObjectPath(newPath) - } - } - } -} - -func traverseArrays(requestArray []interface{}, responseArray []interface{}, path []interface{}) { - for i, val := range requestArray { - newPath := append(path, i) - if len(responseArray) <= i { - // requestArray had more entries than responseArray - break - } - responseVal := responseArray[i] - switch concreteVal := val.(type) { - case map[string]interface{}: - traverseMaps(concreteVal, responseVal.(map[string]interface{}), newPath) - case []interface{}: - traverseArrays(concreteVal, responseVal.([]interface{}), newPath) - case string: - if responseVal == val { - requestArray[i] = jsObjectPath(newPath) - } - default: - panic(jsObjectPath(newPath)) - } - } -} - -func jsObjectPath(path []interface{}) string { - s := "${json" - for _, val := range path { - // this may cause issues with non-array keys with numeric values. test this later. - switch concreteVal := val.(type) { - case int: - s = s + "[" + fmt.Sprint(concreteVal) + "]" - case string: - s = s + "." + concreteVal - } - } - s = s + "}" - return s -} diff --git a/converter/har/converter_test.go b/converter/har/converter_test.go deleted file mode 100644 index 6ef78224848f..000000000000 --- a/converter/har/converter_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package har - -import ( - "fmt" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - - "go.k6.io/k6/js" - "go.k6.io/k6/lib" - "go.k6.io/k6/lib/testutils" - "go.k6.io/k6/loader" - "go.k6.io/k6/metrics" -) - -func TestBuildK6Headers(t *testing.T) { - headers := []struct { - values []Header - expected []string - }{ - {[]Header{{"name", "1"}, {"name", "2"}}, []string{`"name": "1"`}}, - {[]Header{{"name", "1"}, {"name2", "2"}}, []string{`"name": "1"`, `"name2": "2"`}}, - {[]Header{{":host", "localhost"}}, []string{}}, - } - - for _, pair := range headers { - v := buildK6Headers(pair.values) - assert.Equal(t, len(v), len(pair.expected), fmt.Sprintf("params: %v", pair.values)) - } -} - -func TestBuildK6RequestObject(t *testing.T) { - req := &Request{ - Method: "get", - URL: "http://www.google.es", - Headers: []Header{{"accept-language", "es-ES,es;q=0.8"}}, - Cookies: []Cookie{{Name: "a", Value: "b"}}, - } - v, err := buildK6RequestObject(req) - assert.NoError(t, err) - registry := metrics.NewRegistry() - builtinMetrics := metrics.RegisterBuiltinMetrics(registry) - _, err = js.New( - &lib.TestPreInitState{ - Logger: testutils.NewLogger(t), - BuiltinMetrics: builtinMetrics, - Registry: registry, - }, &loader.SourceData{ - URL: &url.URL{Path: "/script.js"}, - Data: []byte(fmt.Sprintf("export default function() { res = http.batch([%v]); }", v)), - }, nil) - assert.NoError(t, err) -} - -func TestBuildK6Body(t *testing.T) { - bodyText := "ccustemail=ppcano%40gmail.com&size=medium&topping=cheese&delivery=12%3A00&comments=" - - req := &Request{ - Method: "post", - URL: "http://www.google.es", - PostData: &PostData{ - MimeType: "application/x-www-form-urlencoded", - Text: bodyText, - }, - } - postParams, plainText, err := buildK6Body(req) - assert.NoError(t, err) - assert.Equal(t, len(postParams), 0, "postParams should be empty") - assert.Equal(t, bodyText, plainText) - - email := "user@mail.es" - expectedEmailParam := fmt.Sprintf(`"email": %q`, email) - - req = &Request{ - Method: "post", - URL: "http://www.google.es", - PostData: &PostData{ - MimeType: "application/x-www-form-urlencoded", - Params: []Param{ - {Name: "email", Value: url.QueryEscape(email)}, - {Name: "pw", Value: "hola"}, - }, - }, - } - postParams, plainText, err = buildK6Body(req) - assert.NoError(t, err) - assert.Equal(t, plainText, "", "expected empty plainText") - assert.Equal(t, len(postParams), 2, "postParams should have two items") - assert.Equal(t, postParams[0], expectedEmailParam, "expected unescaped value") -} diff --git a/converter/har/types.go b/converter/har/types.go deleted file mode 100644 index 7a64743bae3b..000000000000 --- a/converter/har/types.go +++ /dev/null @@ -1,234 +0,0 @@ -package har - -import ( - "time" -) - -// HAR is the top level object of a HAR log. -type HAR struct { - Log *Log `json:"log"` -} - -// Log is the HAR HTTP request and response log. -type Log struct { - // Version number of the HAR format. - Version string `json:"version"` - // Creator holds information about the log creator application. - Creator *Creator `json:"creator"` - // Browser - Browser *Browser `json:"browser,omitempty"` - // Pages - Pages []Page `json:"pages,omitempty"` - // Entries is a list containing requests and responses. - Entries []*Entry `json:"entries"` - // - Comment string `json:"comment,omitempty"` -} - -// Creator is the program responsible for generating the log. Martian, in this case. -type Creator struct { - // Name of the log creator application. - Name string `json:"name"` - // Version of the log creator application. - Version string `json:"version"` -} - -// Browser that created the log -type Browser struct { - // Required. The name of the browser that created the log. - Name string `json:"name"` - // Required. The version number of the browser that created the log. - Version string `json:"version"` - // Optional. A comment provided by the user or the browser. - Comment string `json:"comment"` -} - -// Page object for every exported web page and one object for every HTTP request. -// In case when an HTTP trace tool isn't able to group requests by a page, -// the object is empty and individual requests doesn't have a parent page. -type Page struct { - /* There is one object for every exported web page and one - object for every HTTP request. In case when an HTTP trace tool isn't able to - group requests by a page, the object is empty and individual - requests doesn't have a parent page. - */ - - // Date and time stamp for the beginning of the page load - // (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00). - StartedDateTime time.Time `json:"startedDateTime"` - // Unique identifier of a page within the . Entries use it to refer the parent page. - ID string `json:"id"` - // Page title. - Title string `json:"title"` - // (new in 1.2) A comment provided by the user or the application. - Comment string `json:"comment,omitempty"` -} - -// Entry is a individual log entry for a request or response. -type Entry struct { - Pageref string `json:"pageref,omitempty"` - // ID is the unique ID for the entry. - ID string `json:"_id"` - // StartedDateTime is the date and time stamp of the request start (ISO 8601). - StartedDateTime time.Time `json:"startedDateTime"` - // Time is the total elapsed time of the request in milliseconds. - Time float32 `json:"time"` - // Request contains the detailed information about the request. - Request *Request `json:"request"` - // Response contains the detailed information about the response. - Response *Response `json:"response,omitempty"` - // Cache contains information about a request coming from browser cache. - Cache *Cache `json:"cache"` - // Timings describes various phases within request-response round trip. All - // times are specified in milliseconds. - Timings *Timings `json:"timings"` -} - -// Request holds data about an individual HTTP request. -type Request struct { - // Method is the request method (GET, POST, ...). - Method string `json:"method"` - // URL is the absolute URL of the request (fragments are not included). - URL string `json:"url"` - // HTTPVersion is the Request HTTP version (HTTP/1.1). - HTTPVersion string `json:"httpVersion"` - // Cookies is a list of cookies. - Cookies []Cookie `json:"cookies"` - // Headers is a list of headers. - Headers []Header `json:"headers"` - // QueryString is a list of query parameters. - QueryString []QueryString `json:"queryString"` - // PostData is the posted data information. - PostData *PostData `json:"postData,omitempty"` - // HeaderSize is the Total number of bytes from the start of the HTTP request - // message until (and including) the double CLRF before the body. Set to -1 - // if the info is not available. - HeadersSize int64 `json:"headersSize"` - // BodySize is the size of the request body (POST data payload) in bytes. Set - // to -1 if the info is not available. - BodySize int64 `json:"bodySize"` - // (new in 1.2) A comment provided by the user or the application. - Comment string `json:"comment"` -} - -// Response holds data about an individual HTTP response. -type Response struct { - // Status is the response status code. - Status int `json:"status"` - // StatusText is the response status description. - StatusText string `json:"statusText"` - // HTTPVersion is the Response HTTP version (HTTP/1.1). - HTTPVersion string `json:"httpVersion"` - // Cookies is a list of cookies. - Cookies []Cookie `json:"cookies"` - // Headers is a list of headers. - Headers []Header `json:"headers"` - // Content contains the details of the response body. - Content *Content `json:"content"` - // RedirectURL is the target URL from the Location response header. - RedirectURL string `json:"redirectURL"` - // HeadersSize is the total number of bytes from the start of the HTTP - // request message until (and including) the double CLRF before the body. - // Set to -1 if the info is not available. - HeadersSize int64 `json:"headersSize"` - // BodySize is the size of the request body (POST data payload) in bytes. Set - // to -1 if the info is not available. - BodySize int64 `json:"bodySize"` -} - -// Cache contains information about a request coming from browser cache. -type Cache struct { - // Has no fields as they are not supported, but HAR requires the "cache" - // object to exist. -} - -// Timings describes various phases within request-response round trip. All -// times are specified in milliseconds -type Timings struct { - // Send is the time required to send HTTP request to the server. - Send float32 `json:"send"` - // Wait is the time spent waiting for a response from the server. - Wait float32 `json:"wait"` - // Receive is the time required to read entire response from server or cache. - Receive float32 `json:"receive"` -} - -// Cookie is the data about a cookie on a request or response. -type Cookie struct { - // Name is the cookie name. - Name string `json:"name"` - // Value is the cookie value. - Value string `json:"value"` - // Path is the path pertaining to the cookie. - Path string `json:"path,omitempty"` - // Domain is the host of the cookie. - Domain string `json:"domain,omitempty"` - // Expires contains cookie expiration time. - Expires time.Time `json:"-"` - // Expires8601 contains cookie expiration time in ISO 8601 format. - Expires8601 string `json:"expires,omitempty"` - // HTTPOnly is set to true if the cookie is HTTP only, false otherwise. - HTTPOnly bool `json:"httpOnly,omitempty"` - // Secure is set to true if the cookie was transmitted over SSL, false - // otherwise. - Secure bool `json:"secure,omitempty"` -} - -// Header is an HTTP request or response header. -type Header struct { - // Name is the header name. - Name string `json:"name"` - // Value is the header value. - Value string `json:"value"` -} - -// QueryString is a query string parameter on a request. -type QueryString struct { - // Name is the query parameter name. - Name string `json:"name"` - // Value is the query parameter value. - Value string `json:"value"` -} - -// PostData describes posted data on a request. -type PostData struct { - // MimeType is the MIME type of the posted data. - MimeType string `json:"mimeType"` - // Params is a list of posted parameters (in case of URL encoded parameters). - Params []Param `json:"params"` - // Text contains the plain text posted data. - Text string `json:"text"` -} - -// Param describes an individual posted parameter. -type Param struct { - // Name of the posted parameter. - Name string `json:"name"` - // Value of the posted parameter. - Value string `json:"value,omitempty"` - // Filename of a posted file. - Filename string `json:"fileName,omitempty"` - // ContentType is the content type of a posted file. - ContentType string `json:"contentType,omitempty"` -} - -// Content describes details about response content. -type Content struct { - // Size is the length of the returned content in bytes. Should be equal to - // response.bodySize if there is no compression and bigger when the content - // has been compressed. - Size int64 `json:"size"` - // MimeType is the MIME type of the response text (value of the Content-Type - // response header). - MimeType string `json:"mimeType"` - // Text contains the response body sent from the server or loaded from the - // browser cache. This field is populated with textual content only. The text - // field is either HTTP decoded text or a encoded (e.g. "base64") - // representation of the response body. Leave out this field if the - // information is not available. - Text string `json:"text,omitempty"` - // Encoding used for response text field e.g "base64". Leave out this field - // if the text field is HTTP decoded (decompressed & unchunked), than - // trans-coded from its original character set into UTF-8. - Encoding string `json:"encoding,omitempty"` -} diff --git a/converter/har/utils.go b/converter/har/utils.go deleted file mode 100644 index d223f3a07304..000000000000 --- a/converter/har/utils.go +++ /dev/null @@ -1,85 +0,0 @@ -package har - -import ( - "encoding/json" - "io" - "strings" - "time" -) - -// Define new types to sort -type EntryByStarted []*Entry - -func (e EntryByStarted) Len() int { return len(e) } - -func (e EntryByStarted) Swap(i, j int) { e[i], e[j] = e[j], e[i] } - -func (e EntryByStarted) Less(i, j int) bool { - return e[i].StartedDateTime.Before(e[j].StartedDateTime) -} - -type PageByStarted []Page - -func (e PageByStarted) Len() int { return len(e) } - -func (e PageByStarted) Swap(i, j int) { e[i], e[j] = e[j], e[i] } - -func (e PageByStarted) Less(i, j int) bool { - return e[i].StartedDateTime.Before(e[j].StartedDateTime) -} - -func Decode(r io.Reader) (HAR, error) { - var har HAR - if err := json.NewDecoder(r).Decode(&har); err != nil { - return HAR{}, err - } - - return har, nil -} - -// Returns true if the given url is allowed from the only (only domains) and skip (skip domains) values, otherwise false -func IsAllowedURL(url string, only, skip []string) bool { - if len(only) != 0 { - for _, v := range only { - v = strings.Trim(v, " ") - if v != "" && strings.Contains(url, v) { - return true - } - } - return false - } - if len(skip) != 0 { - for _, v := range skip { - v = strings.Trim(v, " ") - if v != "" && strings.Contains(url, v) { - return false - } - } - } - return true -} - -func SplitEntriesInBatches(entries []*Entry, interval uint) [][]*Entry { - var r [][]*Entry - r = append(r, []*Entry{}) - - if interval > 0 && len(entries) > 1 { - j := 0 - d := time.Duration(interval) * time.Millisecond - for i, e := range entries { - - if i != 0 { - prev := entries[i-1] - if e.StartedDateTime.Sub(prev.StartedDateTime) >= d { - r = append(r, []*Entry{}) - j++ - } - } - r[j] = append(r[j], e) - } - } else { - r[0] = entries - } - - return r -} diff --git a/converter/har/utils_test.go b/converter/har/utils_test.go deleted file mode 100644 index e97ac10edb97..000000000000 --- a/converter/har/utils_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package har - -import ( - "fmt" - "sort" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestIsAllowedURL(t *testing.T) { - allowed := []struct { - url string - only []string - skip []string - expected bool - }{ - {"http://www.google.com/", []string{}, []string{}, true}, - {"http://www.google.com/", []string{"google.com"}, []string{}, true}, - {"https://www.google.com/", []string{"google.com"}, []string{}, true}, - {"https://www.google.com/", []string{"http://"}, []string{}, false}, - {"http://www.google.com/?hl=en", []string{"http://www.google.com"}, []string{}, true}, - {"http://www.google.com/?hl=en", []string{"google.com", "google.co.uk"}, []string{}, true}, - {"http://www.google.com/?hl=en", []string{}, []string{"google.com"}, false}, - {"http://www.google.com/?hl=en", []string{}, []string{"google.co.uk"}, true}, - } - - for _, s := range allowed { - v := IsAllowedURL(s.url, s.only, s.skip) - assert.Equal(t, v, s.expected, fmt.Sprintf("params: %v, %v, %v", s.url, s.only, s.skip)) - } -} - -func TestSplitEntriesInBatches(t *testing.T) { - t1 := time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) - - entries := []*Entry{} - - // 10 time entries with increments of 100ms or 200ms - for i := 1; i <= 10; i++ { - - period := 100 - if i%2 == 0 { - period = 200 - } - t1 = t1.Add(time.Duration(period) * time.Millisecond) - entries = append(entries, &Entry{StartedDateTime: t1}) - } - - splitValues := []struct { - diff, groups uint - }{ - {0, 1}, - {100, 10}, - {150, 6}, - {200, 6}, - {201, 1}, - {500, 1}, - } - - sort.Sort(EntryByStarted(entries)) - - for _, v := range splitValues { - result := SplitEntriesInBatches(entries, v.diff) - assert.Equal(t, len(result), int(v.groups), fmt.Sprintf("params: entries, %v", v.diff)) - } -} diff --git a/go.mod b/go.mod index f309e37ce3b4..fe194a758123 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd github.com/mstoykov/envconfig v1.4.1-0.20220114105314-765c6d8c76f1 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d - github.com/pmezard/go-difflib v1.0.0 github.com/serenize/snaker v0.0.0-20201027110005-a7ad2135616e github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.1.2 @@ -39,7 +38,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.16.0 - github.com/tidwall/pretty v1.2.1 go.uber.org/goleak v1.2.1 golang.org/x/crypto v0.12.0 golang.org/x/net v0.14.0 @@ -71,11 +69,13 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mstoykov/k6-taskqueue-lib v0.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect