From 338000bcd6d277d9de76021ef49bb1befa598adb Mon Sep 17 00:00:00 2001 From: Simon Kirillov Date: Fri, 30 Aug 2024 11:36:44 +0200 Subject: [PATCH] Add support for exporting report in multiple formats (#256) * Bump Go version * Update Dockerfile * Add support for exporting report in multiple formats --- Dockerfile | 2 +- cmd/gotestwaf/flags.go | 59 ++++++++++--- cmd/gotestwaf/helpers.go | 16 ++++ cmd/gotestwaf/main.go | 11 ++- go.mod | 2 +- go.sum | 8 -- internal/config/config.go | 14 ++-- internal/report/report.go | 129 +++++++++++++++++++++++------ tests/integration/config/config.go | 2 +- 9 files changed, 184 insertions(+), 59 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c6c4b46..41fe8ed4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # Build Stage ================================================================== -FROM golang:1.22-alpine AS build +FROM golang:1.23-alpine AS build RUN apk --no-cache add git diff --git a/cmd/gotestwaf/flags.go b/cmd/gotestwaf/flags.go index 849944c3..3e93a7c1 100644 --- a/cmd/gotestwaf/flags.go +++ b/cmd/gotestwaf/flags.go @@ -2,9 +2,11 @@ package main import ( "fmt" + "maps" "os" "path/filepath" "regexp" + "slices" "strings" "github.com/hashicorp/go-multierror" @@ -16,9 +18,36 @@ import ( "github.com/wallarm/gotestwaf/internal/config" "github.com/wallarm/gotestwaf/internal/helpers" + "github.com/wallarm/gotestwaf/internal/report" "github.com/wallarm/gotestwaf/internal/version" ) +const ( + textLogFormat = "text" + jsonLogFormat = "json" +) + +var ( + logFormatsSet = map[string]any{ + textLogFormat: nil, + jsonLogFormat: nil, + } + logFormats = slices.Collect(maps.Keys(logFormatsSet)) +) + +const ( + chromeClient = "chrome" + gohttpClient = "gohttp" +) + +var ( + httpClientsSet = map[string]any{ + chromeClient: nil, + gohttpClient: nil, + } + httpClients = slices.Collect(maps.Keys(httpClientsSet)) +) + const ( maxReportFilenameLength = 249 // 255 (max length) - 5 (".html") - 1 (to be sure) @@ -28,11 +57,9 @@ const ( defaultConfigPath = "config.yaml" wafName = "generic" +) - textLogFormat = "text" - jsonLogFormat = "json" - - cliDescription = `GoTestWAF is a tool for API and OWASP attack simulation that supports a +const cliDescription = `GoTestWAF is a tool for API and OWASP attack simulation that supports a wide range of API protocols including REST, GraphQL, gRPC, SOAP, XMLRPC, and others. Homepage: https://github.com/wallarm/gotestwaf @@ -40,7 +67,6 @@ Usage: %s [OPTIONS] --url Options: ` -) var ( configPath string @@ -69,7 +95,7 @@ func parseFlags() (args []string, err error) { flag.StringVar(&configPath, "configPath", defaultConfigPath, "Path to the config file") flag.BoolVar(&quiet, "quiet", false, "If true, disable verbose logging") logLvl := flag.String("logLevel", "info", "Logging level: panic, fatal, error, warn, info, debug, trace") - flag.StringVar(&logFormat, "logFormat", textLogFormat, "Set logging format: text, json") + flag.StringVar(&logFormat, "logFormat", textLogFormat, "Set logging format: "+strings.Join(logFormats, ", ")) showVersion := flag.Bool("version", false, "Show GoTestWAF version and exit") // Target settings @@ -84,7 +110,7 @@ func parseFlags() (args []string, err error) { flag.String("testSet", "", "If set then only this test set's cases will be run") // HTTP client settings - httpClient := flag.String("httpClient", "gohttp", "Which HTTP client use to send requests: chrome, gohttp") + httpClient := flag.String("httpClient", gohttpClient, "Which HTTP client use to send requests: "+strings.Join(httpClients, ", ")) flag.Bool("tlsVerify", false, "If true, the received TLS certificate will be verified") flag.String("proxy", "", "Proxy URL to use") flag.String("addHeader", "", "An HTTP header to add to requests") @@ -121,7 +147,7 @@ func parseFlags() (args []string, err error) { flag.Bool("includePayloads", false, "If true, payloads will be included in HTML/PDF report") flag.String("reportPath", reportPath, "A directory to store reports") reportName := flag.String("reportName", defaultReportName, "Report file name. Supports `time' package template format") - flag.String("reportFormat", "pdf", "Export report to one of the following formats: none, pdf, html, json") + reportFormat := flag.StringSlice("reportFormat", []string{report.PdfFormat}, "Export report in the following formats: "+strings.Join(report.ReportFormats, ", ")) noEmailReport := flag.Bool("noEmailReport", false, "Save report locally") email := flag.String("email", "", "E-mail to which the report will be sent") @@ -166,8 +192,16 @@ func parseFlags() (args []string, err error) { } logLevel = logrusLogLvl - if logFormat != textLogFormat && logFormat != jsonLogFormat { - return nil, fmt.Errorf("unknown logging format: %s", logFormat) + if err = validateLogFormat(logFormat); err != nil { + return nil, err + } + + if err = validateHttpClient(*httpClient); err != nil { + return nil, err + } + + if err = report.ValidateReportFormat(*reportFormat); err != nil { + return nil, err } validURL, err := validateURL(*urlParam, httpProto) @@ -261,6 +295,11 @@ func normalizeArgs() ([]string, error) { arg = fmt.Sprintf("--%s=%s", f.Name, value) + case "stringSlice": + // remove square brackets: [pdf,json] -> pdf,json + value = strings.Trim(f.Value.String(), "[]") + arg = fmt.Sprintf("--%s=%s", f.Name, value) + case "bool": arg = fmt.Sprintf("--%s", f.Name) diff --git a/cmd/gotestwaf/helpers.go b/cmd/gotestwaf/helpers.go index b9940bf9..ecfc3cf1 100644 --- a/cmd/gotestwaf/helpers.go +++ b/cmd/gotestwaf/helpers.go @@ -65,3 +65,19 @@ func checkOrCraftProtocolURL(rawURL string, validHttpURL string, protocol string return validURL, nil } + +func validateHttpClient(httpClient string) error { + if _, ok := httpClientsSet[httpClient]; !ok { + return fmt.Errorf("invalid HTTP client: %s", httpClient) + } + + return nil +} + +func validateLogFormat(logFormat string) error { + if _, ok := logFormatsSet[logFormat]; !ok { + return fmt.Errorf("invalid log format: %s", logFormat) + } + + return nil +} diff --git a/cmd/gotestwaf/main.go b/cmd/gotestwaf/main.go index 68c87569..8a7c06e1 100644 --- a/cmd/gotestwaf/main.go +++ b/cmd/gotestwaf/main.go @@ -192,12 +192,12 @@ func run(ctx context.Context, cfg *config.Config, logger *logrus.Logger) error { return err } - if cfg.ReportFormat == report.NoneFormat { + if report.IsNoneReportFormat(cfg.ReportFormat) { return nil } includePayloads := cfg.IncludePayloads - if cfg.ReportFormat == report.HtmlFormat || cfg.ReportFormat == report.PdfFormat { + if report.IsPdfOrHtmlReportFormat(cfg.ReportFormat) { askForPayloads := true // If the cfg.IncludePayloads is already explicitly set by the user OR @@ -219,7 +219,7 @@ func run(ctx context.Context, cfg *config.Config, logger *logrus.Logger) error { } } - reportFile, err = report.ExportFullReport( + reportFiles, err := report.ExportFullReport( ctx, stat, reportFile, reportTime, cfg.WAFName, cfg.URL, cfg.OpenAPIFile, cfg.Args, cfg.IgnoreUnresolved, includePayloads, cfg.ReportFormat, @@ -228,7 +228,10 @@ func run(ctx context.Context, cfg *config.Config, logger *logrus.Logger) error { return errors.Wrap(err, "couldn't export full report") } - logger.WithField("filename", reportFile).Infof("Export full report") + for _, file := range reportFiles { + reportExt := strings.ToUpper(strings.Trim(filepath.Ext(file), ".")) + logger.WithField("filename", file).Infof("Export %s full report", reportExt) + } payloadFiles := filepath.Join(cfg.ReportPath, reportName+".csv") err = db.ExportPayloads(payloadFiles) diff --git a/go.mod b/go.mod index 9f39c42c..9ab1291c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/wallarm/gotestwaf -go 1.22 +go 1.23 require ( github.com/chromedp/cdproto v0.0.0-20240810084448-b931b754e476 diff --git a/go.sum b/go.sum index c766da3e..a7182c63 100644 --- a/go.sum +++ b/go.sum @@ -39,14 +39,9 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/cdproto v0.0.0-20240501202034-ef67d660e9fd h1:5/HXKq8EaAWVmnl6Hnyl4SVq7FF5990DBW6AuTrWtVw= -github.com/chromedp/cdproto v0.0.0-20240501202034-ef67d660e9fd/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/cdproto v0.0.0-20240801214329-3f85d328b335/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/cdproto v0.0.0-20240810084448-b931b754e476 h1:VnjHsRXCRti7Av7E+j4DCha3kf68echfDzQ+wD11SBU= github.com/chromedp/cdproto v0.0.0-20240810084448-b931b754e476/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg= -github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y= github.com/chromedp/chromedp v0.10.0 h1:bRclRYVpMm/UVD76+1HcRW9eV3l58rFfy7AdBvKab1E= github.com/chromedp/chromedp v0.10.0/go.mod h1:ei/1ncZIqXX1YnAYDkxhD4gzBgavMEUu7JCKvztdomE= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= @@ -99,7 +94,6 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -412,8 +406,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/internal/config/config.go b/internal/config/config.go index 5b58e268..5ffcae0a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,13 +47,13 @@ type Config struct { BlockConnReset bool `mapstructure:"blockConnReset"` // Report settings - WAFName string `mapstructure:"wafName"` - IncludePayloads bool `mapstructure:"includePayloads"` - ReportPath string `mapstructure:"reportPath"` - ReportName string `mapstructure:"reportName"` - ReportFormat string `mapstructure:"reportFormat"` - NoEmailReport bool `mapstructure:"noEmailReport"` - Email string `mapstructure:"email"` + WAFName string `mapstructure:"wafName"` + IncludePayloads bool `mapstructure:"includePayloads"` + ReportPath string `mapstructure:"reportPath"` + ReportName string `mapstructure:"reportName"` + ReportFormat []string `mapstructure:"reportFormat"` + NoEmailReport bool `mapstructure:"noEmailReport"` + Email string `mapstructure:"email"` // config.yaml HTTPHeaders map[string]string `mapstructure:"headers"` diff --git a/internal/report/report.go b/internal/report/report.go index a9273602..57086be2 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -3,7 +3,10 @@ package report import ( "context" "fmt" + "maps" "path/filepath" + "slices" + "strings" "time" "github.com/pkg/errors" @@ -16,11 +19,22 @@ const ( consoleReportTextFormat = "text" consoleReportJsonFormat = "json" - +) +const ( + NoneFormat = "none" JsonFormat = "json" HtmlFormat = "html" PdfFormat = "pdf" - NoneFormat = "none" +) + +var ( + ReportFormatsSet = map[string]any{ + NoneFormat: nil, + JsonFormat: nil, + HtmlFormat: nil, + PdfFormat: nil, + } + ReportFormats = slices.Collect(maps.Keys(ReportFormatsSet)) ) func SendReportByEmail( @@ -44,41 +58,102 @@ func SendReportByEmail( func ExportFullReport( ctx context.Context, s *db.Statistics, reportFile string, reportTime time.Time, wafName string, url string, openApiFile string, args []string, ignoreUnresolved bool, - includePayloads bool, format string, -) (fullName string, err error) { + includePayloads bool, formats []string, +) (reportFileNames []string, err error) { _, reportFileName := filepath.Split(reportFile) if len(reportFileName) > maxReportFilenameLength { - return "", errors.New("report filename too long") + return nil, errors.New("report filename too long") } - switch format { - case HtmlFormat: - fullName = reportFile + ".html" - err = printFullReportToHtml(s, fullName, reportTime, wafName, url, openApiFile, args, ignoreUnresolved, includePayloads) - if err != nil { - return "", err + for _, format := range formats { + switch format { + case HtmlFormat: + reportFileName = reportFile + ".html" + err = printFullReportToHtml(s, reportFileName, reportTime, wafName, url, openApiFile, args, ignoreUnresolved, includePayloads) + if err != nil { + return nil, err + } + + case PdfFormat: + reportFileName = reportFile + ".pdf" + err = printFullReportToPdf(ctx, s, reportFileName, reportTime, wafName, url, openApiFile, args, ignoreUnresolved, includePayloads) + if err != nil { + return nil, err + } + + case JsonFormat: + reportFileName = reportFile + ".json" + err = printFullReportToJson(s, reportFileName, reportTime, wafName, url, args, ignoreUnresolved) + if err != nil { + return nil, err + } + + case NoneFormat: + return nil, nil + + default: + return nil, fmt.Errorf("unknown report format: %s", format) } - case PdfFormat: - fullName = reportFile + ".pdf" - err = printFullReportToPdf(ctx, s, fullName, reportTime, wafName, url, openApiFile, args, ignoreUnresolved, includePayloads) - if err != nil { - return "", err - } + reportFileNames = append(reportFileNames, reportFileName) + } + + return reportFileNames, nil +} - case JsonFormat: - fullName = reportFile + ".json" - err = printFullReportToJson(s, fullName, reportTime, wafName, url, args, ignoreUnresolved) - if err != nil { - return "", err +func ValidateReportFormat(formats []string) error { + if len(formats) == 0 { + return errors.New("no report format specified") + } + + // Convert slice to set (map) + set := make(map[string]any) + for _, s := range formats { + if _, ok := ReportFormatsSet[s]; !ok { + return fmt.Errorf("unknown report format: %s", s) } - case NoneFormat: - return "", nil + set[s] = nil + } + + // Check for duplicating values + if len(set) != len(formats) { + return fmt.Errorf("found duplicated values: %s", strings.Join(formats, ",")) + } + + // Check "none" is present + _, isNone := set[NoneFormat] + + // Check for conflicts + if len(set) > 1 && isNone { + // Delete "none" from the set + delete(set, NoneFormat) + // Collect conflicted formats + conflictedFormats := slices.Collect(maps.Keys(set)) - default: - return "", fmt.Errorf("unknown report format: %s", format) + return fmt.Errorf("\"none\" conflicts with other formats: %s", strings.Join(conflictedFormats, ",")) + } + + return nil +} + +func IsNoneReportFormat(reportFormat []string) bool { + if len(reportFormat) > 0 && reportFormat[0] == NoneFormat { + return true + } + + return false +} + +func IsPdfOrHtmlReportFormat(reportFormats []string) bool { + for _, format := range reportFormats { + if format == PdfFormat { + return true + } + if format == HtmlFormat { + return true + } } - return fullName, nil + return false } diff --git a/tests/integration/config/config.go b/tests/integration/config/config.go index b724e228..74d3949d 100644 --- a/tests/integration/config/config.go +++ b/tests/integration/config/config.go @@ -146,7 +146,7 @@ func getConfig(httpPort int, grpcPort int) *config.Config { IncludePayloads: false, ReportPath: path.Join(os.TempDir(), "reports"), ReportName: "test", - ReportFormat: "", + ReportFormat: []string{""}, NoEmailReport: true, Email: "",