diff --git a/internal/experiment/hhfm/hhfm.go b/internal/experiment/hhfm/hhfm.go index 93b8eeb97e..a392497987 100644 --- a/internal/experiment/hhfm/hhfm.go +++ b/internal/experiment/hhfm/hhfm.go @@ -278,7 +278,7 @@ func NewRequestEntryList(req *http.Request, headers map[string]string) (out []tr // specific *http.Response instance and its body. func NewHTTPResponse(resp *http.Response, data []byte) (out tracex.HTTPResponse) { out = tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString(data), + Body: model.ArchivalScrubbedMaybeBinaryString(data), Code: int64(resp.StatusCode), Headers: model.ArchivalNewHTTPHeadersMap(resp.Header), HeadersList: model.ArchivalNewHTTPHeadersList(resp.Header), diff --git a/internal/experiment/hhfm/hhfm_test.go b/internal/experiment/hhfm/hhfm_test.go index 2a254c4282..d14c29cae0 100644 --- a/internal/experiment/hhfm/hhfm_test.go +++ b/internal/experiment/hhfm/hhfm_test.go @@ -744,13 +744,13 @@ func TestNewRequestEntryList(t *testing.T) { wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("ContENt-tYPE"), - model.ArchivalMaybeBinaryString("text/plain"), + model.ArchivalScrubbedMaybeBinaryString("ContENt-tYPE"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, { - model.ArchivalMaybeBinaryString("User-aGENT"), - model.ArchivalMaybeBinaryString("foo/1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-aGENT"), + model.ArchivalScrubbedMaybeBinaryString("foo/1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "ContENt-tYPE": "text/plain", "User-aGENT": "foo/1.0", }, @@ -773,7 +773,7 @@ func TestNewRequestEntryList(t *testing.T) { wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ Method: "GeT", - Headers: make(map[string]model.ArchivalMaybeBinaryString), + Headers: make(map[string]model.ArchivalScrubbedMaybeBinaryString), HeadersList: []model.ArchivalHTTPHeader{}, URL: "http://10.0.0.1/", }, @@ -811,16 +811,16 @@ func TestNewHTTPResponse(t *testing.T) { data: []byte("deadbeef"), }, wantOut: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString("deadbeef"), + Body: model.ArchivalScrubbedMaybeBinaryString("deadbeef"), Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/plain"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("foo/1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("foo/1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/plain", "User-Agent": "foo/1.0", }, @@ -831,10 +831,10 @@ func TestNewHTTPResponse(t *testing.T) { resp: &http.Response{StatusCode: 200}, }, wantOut: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), Code: 200, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, }} for _, tt := range tests { diff --git a/internal/experiment/riseupvpn/riseupvpn_test.go b/internal/experiment/riseupvpn/riseupvpn_test.go index 95b12d8321..aee79313c5 100644 --- a/internal/experiment/riseupvpn/riseupvpn_test.go +++ b/internal/experiment/riseupvpn/riseupvpn_test.go @@ -773,11 +773,11 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st Failure: failure, Request: tracex.HTTPRequest{ URL: url, - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, }, Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( responseBody, ), BodyIsTruncated: false, diff --git a/internal/experiment/webconnectivity/httpanalysis_test.go b/internal/experiment/webconnectivity/httpanalysis_test.go index 77b54cba8a..1f22948568 100644 --- a/internal/experiment/webconnectivity/httpanalysis_test.go +++ b/internal/experiment/webconnectivity/httpanalysis_test.go @@ -77,7 +77,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(768), ), }, @@ -96,7 +96,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(768), ), }, @@ -116,7 +116,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(1024), ), }, @@ -136,7 +136,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(8), ), }, @@ -156,7 +156,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(16), ), }, @@ -366,7 +366,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, @@ -382,7 +382,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, @@ -402,7 +402,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, @@ -425,7 +425,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", "Antani": "MASCETTI", }, @@ -450,7 +450,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", "Antani": "MASCETTI", }, @@ -475,7 +475,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept-Ranges": "bytes", "Age": "404727", "Cache-Control": "max-age=604800", @@ -522,7 +522,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept-Ranges": "bytes", "Age": "404727", "Cache-Control": "max-age=604800", @@ -567,7 +567,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept-Ranges": "bytes", "Age": "404727", "Cache-Control": "max-age=604800", @@ -608,7 +608,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "accept-ranges": "bytes", "AGE": "404727", "cache-Control": "max-age=604800", @@ -699,7 +699,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), }, }}, }, @@ -712,7 +712,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), }, }}, }, @@ -731,7 +731,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La community di MSN"), }, }}, @@ -751,7 +751,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La communità di MSN"), }, }}, @@ -771,7 +771,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "" + randx.Letters(1024) + ""), }, }}, @@ -791,7 +791,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La commUNity di MSN"), }, }}, @@ -811,7 +811,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La commUNity di MSN"), }, }}, diff --git a/internal/legacy/tracex/archival.go b/internal/legacy/tracex/archival.go index 31ebbc6242..0fae83d0a7 100644 --- a/internal/legacy/tracex/archival.go +++ b/internal/legacy/tracex/archival.go @@ -118,7 +118,7 @@ func newRequestList(begin time.Time, events []Event) (out []RequestEntry) { entry.Response.HeadersList = model.ArchivalNewHTTPHeadersList(ev.HTTPResponseHeaders) entry.Response.Code = int64(ev.HTTPStatusCode) entry.Response.Locations = ev.HTTPResponseHeaders.Values("Location") - entry.Response.Body = model.ArchivalMaybeBinaryString(ev.HTTPResponseBody) + entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString(ev.HTTPResponseBody) entry.Response.BodyIsTruncated = ev.HTTPResponseBodyIsTruncated entry.Failure = ev.Err.ToFailure() out = append(out, entry) diff --git a/internal/legacy/tracex/archival_test.go b/internal/legacy/tracex/archival_test.go index 2045f02576..685b5bbf69 100644 --- a/internal/legacy/tracex/archival_test.go +++ b/internal/legacy/tracex/archival_test.go @@ -173,10 +173,10 @@ func TestNewRequestList(t *testing.T) { Failure: NewFailure(io.EOF), Request: HTTPRequest{ HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", @@ -184,30 +184,30 @@ func TestNewRequestList(t *testing.T) { }, Response: HTTPResponse{ HeadersList: []model.ArchivalHTTPHeader{}, - Headers: make(map[string]model.ArchivalMaybeBinaryString), + Headers: make(map[string]model.ArchivalScrubbedMaybeBinaryString), }, T: 0.02, }, { Request: HTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "User-Agent": "miniooni/0.1.0-dev", }, Method: "POST", URL: "https://www.example.com/submit", }, Response: HTTPResponse{ - Body: model.ArchivalMaybeBinaryString("{}"), + Body: model.ArchivalScrubbedMaybeBinaryString("{}"), Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Server": "miniooni/0.1.0-dev", }, Locations: nil, @@ -237,10 +237,10 @@ func TestNewRequestList(t *testing.T) { want: []RequestEntry{{ Request: HTTPRequest{ HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", @@ -249,16 +249,16 @@ func TestNewRequestList(t *testing.T) { Response: HTTPResponse{ Code: 302, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Location"), - model.ArchivalMaybeBinaryString("https://x.example.com"), + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("https://x.example.com"), }, { - model.ArchivalMaybeBinaryString("Location"), - model.ArchivalMaybeBinaryString("https://y.example.com"), + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("https://y.example.com"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Server": "miniooni/0.1.0-dev", "Location": "https://x.example.com", }, diff --git a/internal/measurexlite/http.go b/internal/measurexlite/http.go index 8c196926ce..e595629308 100644 --- a/internal/measurexlite/http.go +++ b/internal/measurexlite/http.go @@ -51,7 +51,7 @@ func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, a ALPN: alpn, Failure: NewFailure(err), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: newHTTPRequestHeaderList(req), Headers: newHTTPRequestHeaderMap(req), @@ -61,7 +61,7 @@ func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, a URL: httpRequestURL(req), }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(body), + Body: model.ArchivalScrubbedMaybeBinaryString(body), BodyIsTruncated: httpResponseBodyIsTruncated(body, maxRespBodySize), Code: httpResponseStatusCode(resp), HeadersList: newHTTPResponseHeaderList(resp), @@ -95,7 +95,7 @@ func newHTTPRequestHeaderList(req *http.Request) []model.ArchivalHTTPHeader { // newHTTPRequestHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalMaybeBinaryString { +func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalScrubbedMaybeBinaryString { m := http.Header{} if req != nil { m = req.Header @@ -139,7 +139,7 @@ func newHTTPResponseHeaderList(resp *http.Response) (out []model.ArchivalHTTPHea // newHTTPResponseHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryString) { +func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalScrubbedMaybeBinaryString) { m := http.Header{} if resp != nil { m = resp.Header diff --git a/internal/measurexlite/http_test.go b/internal/measurexlite/http_test.go index c72abcde4d..ed31374170 100644 --- a/internal/measurexlite/http_test.go +++ b/internal/measurexlite/http_test.go @@ -58,21 +58,21 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Method: "", Tor: model.ArchivalHTTPTor{}, Transport: "", URL: "", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: []string{}, }, T0: 0, @@ -117,16 +117,16 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { return &s }(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("*/*"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "*/*", "User-Agent": "miniooni/0.1.0-dev", }, @@ -136,11 +136,11 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "http://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: []string{}, }, T0: 0.25, @@ -188,16 +188,16 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "h3", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("*/*"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "*/*", "User-Agent": "miniooni/0.1.0-dev", }, @@ -207,19 +207,19 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "https://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( testingx.HTTPBlockpage451, ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/html; charset=iso-8859-1"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=iso-8859-1"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/html; charset=iso-8859-1", "Server": "Apache", }, @@ -278,16 +278,16 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "h3", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("*/*"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "*/*", "User-Agent": "miniooni/0.1.0-dev", }, @@ -297,20 +297,20 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "https://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 302, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/html; charset=iso-8859-1"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=iso-8859-1"), }, { - model.ArchivalMaybeBinaryString("Location"), - model.ArchivalMaybeBinaryString("/v2/index.html"), + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("/v2/index.html"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/html; charset=iso-8859-1", "Location": "/v2/index.html", "Server": "Apache", diff --git a/internal/model/archival.go b/internal/model/archival.go index 514120ec00..474e2163c3 100644 --- a/internal/model/archival.go +++ b/internal/model/archival.go @@ -19,6 +19,8 @@ import ( "net/http" "sort" "unicode/utf8" + + "github.com/ooni/probe-cli/v3/internal/scrubber" ) // @@ -121,23 +123,27 @@ func (value *ArchivalBinaryData) UnmarshalJSON(raw []byte) error { return nil } -// ArchivalMaybeBinaryString is a possibly-binary string. When the string is valid UTF-8 +// ArchivalScrubbedMaybeBinaryString is a possibly-binary string. When the string is valid UTF-8 // we serialize it as itself. Otherwise, we use the binary data format defined by // https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata -type ArchivalMaybeBinaryString string +// +// As the name implies, the data contained by this type is scrubbed to remove IPv4 and IPv6 +// addresses and endpoints during JSON serialization, to make it less likely that OONI leaks +// IP addresses in textual or binary fields such as HTTP headers and bodies. +type ArchivalScrubbedMaybeBinaryString string var ( - _ json.Marshaler = ArchivalMaybeBinaryString("") - _ json.Unmarshaler = (func() *ArchivalMaybeBinaryString { return nil }()) + _ json.Marshaler = ArchivalScrubbedMaybeBinaryString("") + _ json.Unmarshaler = (func() *ArchivalScrubbedMaybeBinaryString { return nil }()) ) // MarshalJSON implements json.Marshaler. -func (value ArchivalMaybeBinaryString) MarshalJSON() ([]byte, error) { +func (value ArchivalScrubbedMaybeBinaryString) MarshalJSON() ([]byte, error) { // convert value to a string str := string(value) - // TODO(bassosimone): here is where we should scrub the string in the future - // once we have replaced AchivalMaybeBinaryData with ArchivalMaybeBinaryString + // make sure we get rid of IPv4 and IPv6 addresses and endpoints + str = scrubber.ScrubString(str) // if we can serialize as UTF-8 string, do that if utf8.ValidString(str) { @@ -149,11 +155,11 @@ func (value ArchivalMaybeBinaryString) MarshalJSON() ([]byte, error) { } // UnmarshalJSON implements json.Unmarshaler. -func (value *ArchivalMaybeBinaryString) UnmarshalJSON(rawData []byte) error { +func (value *ArchivalScrubbedMaybeBinaryString) UnmarshalJSON(rawData []byte) error { // first attempt to decode as a string var s string if err := json.Unmarshal(rawData, &s); err == nil { - *value = ArchivalMaybeBinaryString(s) + *value = ArchivalScrubbedMaybeBinaryString(s) return nil } @@ -162,7 +168,7 @@ func (value *ArchivalMaybeBinaryString) UnmarshalJSON(rawData []byte) error { if err := json.Unmarshal(rawData, &d); err != nil { return err } - *value = ArchivalMaybeBinaryString(d) + *value = ArchivalScrubbedMaybeBinaryString(d) return nil } @@ -173,6 +179,8 @@ func (value *ArchivalMaybeBinaryString) UnmarshalJSON(rawData []byte) error { // See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata // // Deprecated: do not use this type in new code. +// +// Removing this struct is TODO(https://github.com/ooni/probe/issues/2543). type ArchivalMaybeBinaryData struct { Value string } @@ -325,14 +333,14 @@ type ArchivalHTTPRequestResult struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPRequest struct { - Body ArchivalMaybeBinaryString `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryString `json:"headers"` - Method string `json:"method"` - Tor ArchivalHTTPTor `json:"tor"` - Transport string `json:"x_transport"` - URL string `json:"url"` + Body ArchivalScrubbedMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalScrubbedMaybeBinaryString `json:"headers"` + Method string `json:"method"` + Tor ArchivalHTTPTor `json:"tor"` + Transport string `json:"x_transport"` + URL string `json:"url"` } // ArchivalHTTPResponse contains an HTTP response. @@ -340,11 +348,11 @@ type ArchivalHTTPRequest struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPResponse struct { - Body ArchivalMaybeBinaryString `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - Code int64 `json:"code"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryString `json:"headers"` + Body ArchivalScrubbedMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + Code int64 `json:"code"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalScrubbedMaybeBinaryString `json:"headers"` // The following fields are not serialised but are useful to simplify // analysing the measurements in telegram, whatsapp, etc. @@ -370,8 +378,8 @@ func ArchivalNewHTTPHeadersList(source http.Header) (out []ArchivalHTTPHeader) { for _, key := range keys { for _, value := range source[key] { out = append(out, ArchivalHTTPHeader{ - ArchivalMaybeBinaryString(key), - ArchivalMaybeBinaryString(value), + ArchivalScrubbedMaybeBinaryString(key), + ArchivalScrubbedMaybeBinaryString(value), }) } } @@ -379,11 +387,11 @@ func ArchivalNewHTTPHeadersList(source http.Header) (out []ArchivalHTTPHeader) { } // ArchivalNewHTTPHeadersMap creates a map representation of HTTP headers -func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalMaybeBinaryString) { - out = make(map[string]ArchivalMaybeBinaryString) +func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalScrubbedMaybeBinaryString) { + out = make(map[string]ArchivalScrubbedMaybeBinaryString) for key, values := range header { for _, value := range values { - out[key] = ArchivalMaybeBinaryString(value) + out[key] = ArchivalScrubbedMaybeBinaryString(value) break // just the first header } } @@ -391,14 +399,14 @@ func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalMaybe } // ArchivalHTTPHeader is a single HTTP header. -type ArchivalHTTPHeader [2]ArchivalMaybeBinaryString +type ArchivalHTTPHeader [2]ArchivalScrubbedMaybeBinaryString // errCannotParseArchivalHTTPHeader indicates that we cannot parse an ArchivalHTTPHeader. var errCannotParseArchivalHTTPHeader = errors.New("invalid ArchivalHTTPHeader") // UnmarshalJSON implements json.Unmarshaler. func (ahh *ArchivalHTTPHeader) UnmarshalJSON(data []byte) error { - var helper []ArchivalMaybeBinaryString + var helper []ArchivalScrubbedMaybeBinaryString if err := json.Unmarshal(data, &helper); err != nil { return err } diff --git a/internal/model/archival_test.go b/internal/model/archival_test.go index 1979ed7024..6567ce4cfa 100644 --- a/internal/model/archival_test.go +++ b/internal/model/archival_test.go @@ -294,10 +294,10 @@ func TestArchivalBinaryData(t *testing.T) { }) } -func TestArchivalMaybeBinaryString(t *testing.T) { +func TestArchivalScrubbedMaybeBinaryString(t *testing.T) { t.Run("Supports assignment from a nil byte array", func(t *testing.T) { var data []byte = nil // explicit - casted := model.ArchivalMaybeBinaryString(data) + casted := model.ArchivalScrubbedMaybeBinaryString(data) if casted != "" { t.Fatal("unexpected value") } @@ -312,7 +312,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { name string // input is the possibly-binary input - input model.ArchivalMaybeBinaryString + input model.ArchivalScrubbedMaybeBinaryString // expectErr is the error we expect to see or nil expectErr error @@ -333,9 +333,14 @@ func TestArchivalMaybeBinaryString(t *testing.T) { expectData: []byte(`"Elliot"`), }, { name: "with value being a long binary string", - input: model.ArchivalMaybeBinaryString(archivalBinaryInput), + input: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), expectErr: nil, expectData: archivalEncodedBinaryInput, + }, { + name: "with string containing IP addresses and endpoints", + input: "a 130.192.91.211 b ::1 c [::1]:443 d 130.192.91.211:80", + expectErr: nil, + expectData: []byte(`"a [scrubbed] b [scrubbed] c [scrubbed] d [scrubbed]"`), }} for _, tc := range cases { @@ -386,85 +391,85 @@ func TestArchivalMaybeBinaryString(t *testing.T) { expectErr error // expectData is the data we expect - expectData model.ArchivalMaybeBinaryString + expectData model.ArchivalScrubbedMaybeBinaryString } cases := []testcase{{ name: "with nil input array", input: nil, expectErr: errors.New("unexpected end of JSON input"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with zero-length input array", input: []byte{}, expectErr: errors.New("unexpected end of JSON input"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with binary input that is not a complete JSON", input: []byte("{"), expectErr: errors.New("unexpected end of JSON input"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with ~random binary data as input", input: archivalBinaryInput, expectErr: errors.New("invalid character 'W' looking for beginning of value"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with valid JSON of the wrong type (array)", input: []byte("[]"), expectErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with valid JSON of the wrong type (number)", input: []byte("1.17"), expectErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with input being the liternal null", input: []byte(`null`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with empty JSON object", input: []byte("{}"), expectErr: errors.New("model: invalid binary data format: ''"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with correct data model but invalid format", input: []byte(`{"data":"","format":"antani"}`), expectErr: errors.New("model: invalid binary data format: 'antani'"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with correct data model and format but invalid base64 string", input: []byte(`{"data":"x","format":"base64"}`), expectErr: errors.New("illegal base64 data at input byte 0"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with correct data model and format but empty base64 string", input: []byte(`{"data":"","format":"base64"}`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with the a string", input: []byte(`"Elliot"`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString("Elliot"), + expectData: model.ArchivalScrubbedMaybeBinaryString("Elliot"), }, { name: "with the encoding of a string", input: []byte(`{"data":"RWxsaW90","format":"base64"}`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString("Elliot"), + expectData: model.ArchivalScrubbedMaybeBinaryString("Elliot"), }, { name: "with the encoding of a complex binary string", input: archivalEncodedBinaryInput, expectErr: nil, - expectData: model.ArchivalMaybeBinaryString(archivalBinaryInput), + expectData: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }} for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { // unmarshal the raw input into an ArchivalBinaryData type - var abd model.ArchivalMaybeBinaryString + var abd model.ArchivalScrubbedMaybeBinaryString err := json.Unmarshal(tc.input, &abd) t.Log("got this error", err) @@ -503,7 +508,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { name string // input is the maybe-binary input - input model.ArchivalMaybeBinaryString + input model.ArchivalScrubbedMaybeBinaryString } cases := []testcase{{ @@ -514,7 +519,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { input: "Elliot", }, { name: "with value being a long binary string", - input: model.ArchivalMaybeBinaryString(archivalBinaryInput), + input: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }} for _, tc := range cases { @@ -531,7 +536,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { } // parse from JSON - var abc model.ArchivalMaybeBinaryString + var abc model.ArchivalScrubbedMaybeBinaryString if err := json.Unmarshal(output, &abc); err != nil { t.Fatal(err) } @@ -640,16 +645,16 @@ func TestArchivalHTTPHeader(t *testing.T) { }{{ name: "with string value", input: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/plain"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, want: []byte(`["Content-Type","text/plain"]`), wantErr: false, }, { name: "with binary value", input: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString(archivalBinaryInput), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }, want: []byte(`["Content-Type",` + string(archivalEncodedBinaryInput) + `]`), wantErr: false, @@ -677,98 +682,98 @@ func TestArchivalHTTPHeader(t *testing.T) { name: "with invalid input", input: []byte(`{}`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: errors.New("json: cannot unmarshal object into Go value of type []model.ArchivalMaybeBinaryString"), + wantErr: errors.New("json: cannot unmarshal object into Go value of type []model.ArchivalScrubbedMaybeBinaryString"), }, { name: "with zero items", input: []byte(`[]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 0"), }, { name: "with just one item", input: []byte(`["x"]`), - want: [2]model.ArchivalMaybeBinaryString{}, + want: [2]model.ArchivalScrubbedMaybeBinaryString{}, wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 1"), }, { name: "with three items", input: []byte(`["x","x","x"]`), - want: [2]model.ArchivalMaybeBinaryString{}, + want: [2]model.ArchivalScrubbedMaybeBinaryString{}, wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 3"), }, { name: "with first item not being a string", input: []byte(`[0,0]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), }, { name: "with both items being a string", input: []byte(`["x","y"]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("x"), - model.ArchivalMaybeBinaryString("y"), + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString("y"), }, wantErr: nil, }, { name: "with second item not being a map[string]interface{}", input: []byte(`["x",[]]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), }, { name: "with missing format key in second item", input: []byte(`["x",{}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("model: invalid binary data format: ''"), }, { name: "with format value not being base64", input: []byte(`["x",{"format":1}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal number into Go struct field archivalBinaryDataRepr.format of type string"), }, { name: "with missing data field", input: []byte(`["x",{"format":"base64"}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("x"), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: nil, }, { name: "with data not being a string", input: []byte(`["x",{"format":"base64","data":1}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal number into Go struct field archivalBinaryDataRepr.data of type []uint8"), }, { name: "with data not being base64", input: []byte(`["x",{"format":"base64","data":"xx"}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("illegal base64 data at input byte 0"), }, { name: "with correctly encoded base64 data", input: []byte(`["x",` + string(archivalEncodedBinaryInput) + `]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("x"), - model.ArchivalMaybeBinaryString(archivalBinaryInput), + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }, wantErr: nil, }} @@ -1453,16 +1458,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1476,19 +1481,19 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "Bonsoir, Elliot!", ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", }, @@ -1515,16 +1520,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { return &s })(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1538,11 +1543,11 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: nil, }, T0: 0.4, @@ -1567,25 +1572,25 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }, { - model.ArchivalMaybeBinaryString("Antani"), - model.ArchivalMaybeBinaryString(string(archivalBinaryInput[:7])), + model.ArchivalScrubbedMaybeBinaryString("Antani"), + model.ArchivalScrubbedMaybeBinaryString(string(archivalBinaryInput[:7])), }, { - model.ArchivalMaybeBinaryString("Antani"), - model.ArchivalMaybeBinaryString(archivalBinaryInput[7:14]), + model.ArchivalScrubbedMaybeBinaryString("Antani"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[7:14]), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", - "Antani": model.ArchivalMaybeBinaryString(archivalBinaryInput[:7]), + "Antani": model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[:7]), }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1597,28 +1602,28 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( archivalBinaryInput[:77], ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }, { - model.ArchivalMaybeBinaryString("Mascetti"), - model.ArchivalMaybeBinaryString(archivalBinaryInput[14:21]), + model.ArchivalScrubbedMaybeBinaryString("Mascetti"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[14:21]), }, { - model.ArchivalMaybeBinaryString("Mascetti"), - model.ArchivalMaybeBinaryString(archivalBinaryInput[21:28]), + model.ArchivalScrubbedMaybeBinaryString("Mascetti"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[21:28]), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", - "Mascetti": model.ArchivalMaybeBinaryString(archivalEncodedBinaryInput[14:21]), + "Mascetti": model.ArchivalScrubbedMaybeBinaryString(archivalEncodedBinaryInput[14:21]), }, Locations: nil, }, @@ -1649,28 +1654,28 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }, { - model.ArchivalMaybeBinaryString("AntaniV4"), - model.ArchivalMaybeBinaryString("130.192.91.211"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV4"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211"), }, { - model.ArchivalMaybeBinaryString("AntaniV6"), - model.ArchivalMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV6"), + model.ArchivalScrubbedMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), }, { - model.ArchivalMaybeBinaryString("AntaniV4Epnt"), - model.ArchivalMaybeBinaryString("[130.192.91.211]:443"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV4Epnt"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211:443"), }, { - model.ArchivalMaybeBinaryString("AntaniV6Epnt"), - model.ArchivalMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV6Epnt"), + model.ArchivalScrubbedMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", "AntaniV4": "130.192.91.211", @@ -1688,31 +1693,31 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "Your address is 130.192.91.211 and 2606:2800:220:1:248:1893:25c8:1946 and you have endpoints [2606:2800:220:1:248:1893:25c8:1946]:5222 and 130.192.91.211:443. You're welcome.", ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }, { - model.ArchivalMaybeBinaryString("MascettiV4"), - model.ArchivalMaybeBinaryString("130.192.91.211"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV4"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211"), }, { - model.ArchivalMaybeBinaryString("MascettiV6"), - model.ArchivalMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV6"), + model.ArchivalScrubbedMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), }, { - model.ArchivalMaybeBinaryString("MascettiV4Epnt"), - model.ArchivalMaybeBinaryString("[130.192.91.211]:443"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV4Epnt"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211:443"), }, { - model.ArchivalMaybeBinaryString("MascettiV6Epnt"), - model.ArchivalMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV6Epnt"), + model.ArchivalScrubbedMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", "MascettiV4": "130.192.91.211", @@ -1728,7 +1733,7 @@ func TestArchivalHTTPRequestResult(t *testing.T) { TransactionID: 5, }, expectErr: nil, - expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"],["AntaniV4","130.192.91.211"],["AntaniV6","2606:2800:220:1:248:1893:25c8:1946"],["AntaniV4Epnt","[130.192.91.211]:443"],["AntaniV6Epnt","[2606:2800:220:1:248:1893:25c8:1946]:5222"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","AntaniV4":"130.192.91.211","AntaniV4Epnt":"130.192.91.211:443","AntaniV6":"2606:2800:220:1:248:1893:25c8:1946","AntaniV6Epnt":"[2606:2800:220:1:248:1893:25c8:1946]:5222","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"\u003cHTML\u003e\u003cBODY\u003eYour address is 130.192.91.211 and 2606:2800:220:1:248:1893:25c8:1946 and you have endpoints [2606:2800:220:1:248:1893:25c8:1946]:5222 and 130.192.91.211:443. You're welcome.\u003c/BODY\u003e\u003c/HTML\u003e","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"],["MascettiV4","130.192.91.211"],["MascettiV6","2606:2800:220:1:248:1893:25c8:1946"],["MascettiV4Epnt","[130.192.91.211]:443"],["MascettiV6Epnt","[2606:2800:220:1:248:1893:25c8:1946]:5222"]],"headers":{"Age":"131833","MascettiV4":"130.192.91.211","MascettiV4Epnt":"130.192.91.211:443","MascettiV6":"2606:2800:220:1:248:1893:25c8:1946","MascettiV6Epnt":"[2606:2800:220:1:248:1893:25c8:1946]:5222","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), + expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"],["AntaniV4","[scrubbed]"],["AntaniV6","[scrubbed]"],["AntaniV4Epnt","[scrubbed]"],["AntaniV6Epnt","[scrubbed]"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","AntaniV4":"[scrubbed]","AntaniV4Epnt":"[scrubbed]","AntaniV6":"[scrubbed]","AntaniV6Epnt":"[scrubbed]","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"\u003cHTML\u003e\u003cBODY\u003eYour address is [scrubbed] and [scrubbed] and you have endpoints [scrubbed] and [scrubbed]. You're welcome.\u003c/BODY\u003e\u003c/HTML\u003e","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"],["MascettiV4","[scrubbed]"],["MascettiV6","[scrubbed]"],["MascettiV4Epnt","[scrubbed]"],["MascettiV6Epnt","[scrubbed]"]],"headers":{"Age":"131833","MascettiV4":"[scrubbed]","MascettiV4Epnt":"[scrubbed]","MascettiV6":"[scrubbed]","MascettiV6Epnt":"[scrubbed]","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), }, } @@ -1793,16 +1798,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1816,19 +1821,19 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "Bonsoir, Elliot!", ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", }, @@ -1852,16 +1857,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { return &s })(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1875,11 +1880,11 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: nil, }, T0: 0.4, @@ -2125,20 +2130,20 @@ func TestArchivalNewHTTPHeadersList(t *testing.T) { "User-Agent": {"miniooni/0.1.0"}, }, expect: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/html; charset=utf-8"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=utf-8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }, { - model.ArchivalMaybeBinaryString("Via"), - model.ArchivalMaybeBinaryString("a"), + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("a"), }, { - model.ArchivalMaybeBinaryString("Via"), - model.ArchivalMaybeBinaryString("b"), + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("b"), }, { - model.ArchivalMaybeBinaryString("Via"), - model.ArchivalMaybeBinaryString("c"), + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("c"), }}, }} @@ -2158,17 +2163,17 @@ func TestArchivalNewHTTPHeadersMap(t *testing.T) { type testcase struct { name string input http.Header - expect map[string]model.ArchivalMaybeBinaryString + expect map[string]model.ArchivalScrubbedMaybeBinaryString } cases := []testcase{{ name: "with nil input", input: nil, - expect: map[string]model.ArchivalMaybeBinaryString{}, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, { name: "with empty input", input: map[string][]string{}, - expect: map[string]model.ArchivalMaybeBinaryString{}, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, { name: "common case", input: map[string][]string{ @@ -2176,7 +2181,7 @@ func TestArchivalNewHTTPHeadersMap(t *testing.T) { "Via": {"a", "b", "c"}, "User-Agent": {"miniooni/0.1.0"}, }, - expect: map[string]model.ArchivalMaybeBinaryString{ + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/html; charset=utf-8", "Via": "a", "User-Agent": "miniooni/0.1.0",