From 7a6fd910df7bbfd11f9f257cbd778f686d2994ca Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 28 Sep 2023 23:14:48 +0200 Subject: [PATCH] feat(archival): unconditionally scrub HTTP headers and bodies (#1335) This commit modifies ArchivalMaybeBinaryData to unconditionally scrub IP addresses and endpoints. It also renames the struct ArchivalScrubbedMaybeBinaryData to underline that it's scrubbing now. We're using ArchivalScrubbedMaybeBinaryData to represent HTTP headers and bodies collected by OONI measurements. With this commit merged, I am now much less concerned about the potential unintended effects of aggressively using happy eyeballs, which we started introducing as part of https://github.com/ooni/probe/issues/2531. In other words, the rest of the refactoring of the OONI bootstrap could proceed a bit faster. --- internal/experiment/hhfm/hhfm.go | 2 +- internal/experiment/hhfm/hhfm_test.go | 28 +- .../experiment/riseupvpn/riseupvpn_test.go | 4 +- .../webconnectivity/httpanalysis_test.go | 42 +-- internal/legacy/tracex/archival.go | 2 +- internal/legacy/tracex/archival_test.go | 44 +-- internal/measurexlite/http.go | 8 +- internal/measurexlite/http_test.go | 76 ++--- internal/model/archival.go | 68 ++-- internal/model/archival_test.go | 319 +++++++++--------- 10 files changed, 303 insertions(+), 290 deletions(-) diff --git a/internal/experiment/hhfm/hhfm.go b/internal/experiment/hhfm/hhfm.go index 93b8eeb97e..a392497987 100644 --- a/internal/experiment/hhfm/hhfm.go +++ b/internal/experiment/hhfm/hhfm.go @@ -278,7 +278,7 @@ func NewRequestEntryList(req *http.Request, headers map[string]string) (out []tr // specific *http.Response instance and its body. func NewHTTPResponse(resp *http.Response, data []byte) (out tracex.HTTPResponse) { out = tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString(data), + Body: model.ArchivalScrubbedMaybeBinaryString(data), Code: int64(resp.StatusCode), Headers: model.ArchivalNewHTTPHeadersMap(resp.Header), HeadersList: model.ArchivalNewHTTPHeadersList(resp.Header), diff --git a/internal/experiment/hhfm/hhfm_test.go b/internal/experiment/hhfm/hhfm_test.go index 2a254c4282..d14c29cae0 100644 --- a/internal/experiment/hhfm/hhfm_test.go +++ b/internal/experiment/hhfm/hhfm_test.go @@ -744,13 +744,13 @@ func TestNewRequestEntryList(t *testing.T) { wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("ContENt-tYPE"), - model.ArchivalMaybeBinaryString("text/plain"), + model.ArchivalScrubbedMaybeBinaryString("ContENt-tYPE"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, { - model.ArchivalMaybeBinaryString("User-aGENT"), - model.ArchivalMaybeBinaryString("foo/1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-aGENT"), + model.ArchivalScrubbedMaybeBinaryString("foo/1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "ContENt-tYPE": "text/plain", "User-aGENT": "foo/1.0", }, @@ -773,7 +773,7 @@ func TestNewRequestEntryList(t *testing.T) { wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ Method: "GeT", - Headers: make(map[string]model.ArchivalMaybeBinaryString), + Headers: make(map[string]model.ArchivalScrubbedMaybeBinaryString), HeadersList: []model.ArchivalHTTPHeader{}, URL: "http://10.0.0.1/", }, @@ -811,16 +811,16 @@ func TestNewHTTPResponse(t *testing.T) { data: []byte("deadbeef"), }, wantOut: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString("deadbeef"), + Body: model.ArchivalScrubbedMaybeBinaryString("deadbeef"), Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/plain"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("foo/1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("foo/1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/plain", "User-Agent": "foo/1.0", }, @@ -831,10 +831,10 @@ func TestNewHTTPResponse(t *testing.T) { resp: &http.Response{StatusCode: 200}, }, wantOut: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), Code: 200, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, }} for _, tt := range tests { diff --git a/internal/experiment/riseupvpn/riseupvpn_test.go b/internal/experiment/riseupvpn/riseupvpn_test.go index 95b12d8321..aee79313c5 100644 --- a/internal/experiment/riseupvpn/riseupvpn_test.go +++ b/internal/experiment/riseupvpn/riseupvpn_test.go @@ -773,11 +773,11 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st Failure: failure, Request: tracex.HTTPRequest{ URL: url, - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, }, Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( responseBody, ), BodyIsTruncated: false, diff --git a/internal/experiment/webconnectivity/httpanalysis_test.go b/internal/experiment/webconnectivity/httpanalysis_test.go index 77b54cba8a..1f22948568 100644 --- a/internal/experiment/webconnectivity/httpanalysis_test.go +++ b/internal/experiment/webconnectivity/httpanalysis_test.go @@ -77,7 +77,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(768), ), }, @@ -96,7 +96,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(768), ), }, @@ -116,7 +116,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(1024), ), }, @@ -136,7 +136,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(8), ), }, @@ -156,7 +156,7 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( randx.Letters(16), ), }, @@ -366,7 +366,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, @@ -382,7 +382,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, @@ -402,7 +402,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, @@ -425,7 +425,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", "Antani": "MASCETTI", }, @@ -450,7 +450,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Date": "Mon Jul 13 21:10:08 CEST 2020", "Antani": "MASCETTI", }, @@ -475,7 +475,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept-Ranges": "bytes", "Age": "404727", "Cache-Control": "max-age=604800", @@ -522,7 +522,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept-Ranges": "bytes", "Age": "404727", "Cache-Control": "max-age=604800", @@ -567,7 +567,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept-Ranges": "bytes", "Age": "404727", "Cache-Control": "max-age=604800", @@ -608,7 +608,7 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "accept-ranges": "bytes", "AGE": "404727", "cache-Control": "max-age=604800", @@ -699,7 +699,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), }, }}, }, @@ -712,7 +712,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), }, }}, }, @@ -731,7 +731,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La community di MSN"), }, }}, @@ -751,7 +751,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La communità di MSN"), }, }}, @@ -771,7 +771,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "" + randx.Letters(1024) + ""), }, }}, @@ -791,7 +791,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La commUNity di MSN"), }, }}, @@ -811,7 +811,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "La commUNity di MSN"), }, }}, diff --git a/internal/legacy/tracex/archival.go b/internal/legacy/tracex/archival.go index 31ebbc6242..0fae83d0a7 100644 --- a/internal/legacy/tracex/archival.go +++ b/internal/legacy/tracex/archival.go @@ -118,7 +118,7 @@ func newRequestList(begin time.Time, events []Event) (out []RequestEntry) { entry.Response.HeadersList = model.ArchivalNewHTTPHeadersList(ev.HTTPResponseHeaders) entry.Response.Code = int64(ev.HTTPStatusCode) entry.Response.Locations = ev.HTTPResponseHeaders.Values("Location") - entry.Response.Body = model.ArchivalMaybeBinaryString(ev.HTTPResponseBody) + entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString(ev.HTTPResponseBody) entry.Response.BodyIsTruncated = ev.HTTPResponseBodyIsTruncated entry.Failure = ev.Err.ToFailure() out = append(out, entry) diff --git a/internal/legacy/tracex/archival_test.go b/internal/legacy/tracex/archival_test.go index 2045f02576..685b5bbf69 100644 --- a/internal/legacy/tracex/archival_test.go +++ b/internal/legacy/tracex/archival_test.go @@ -173,10 +173,10 @@ func TestNewRequestList(t *testing.T) { Failure: NewFailure(io.EOF), Request: HTTPRequest{ HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", @@ -184,30 +184,30 @@ func TestNewRequestList(t *testing.T) { }, Response: HTTPResponse{ HeadersList: []model.ArchivalHTTPHeader{}, - Headers: make(map[string]model.ArchivalMaybeBinaryString), + Headers: make(map[string]model.ArchivalScrubbedMaybeBinaryString), }, T: 0.02, }, { Request: HTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "User-Agent": "miniooni/0.1.0-dev", }, Method: "POST", URL: "https://www.example.com/submit", }, Response: HTTPResponse{ - Body: model.ArchivalMaybeBinaryString("{}"), + Body: model.ArchivalScrubbedMaybeBinaryString("{}"), Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Server": "miniooni/0.1.0-dev", }, Locations: nil, @@ -237,10 +237,10 @@ func TestNewRequestList(t *testing.T) { want: []RequestEntry{{ Request: HTTPRequest{ HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", @@ -249,16 +249,16 @@ func TestNewRequestList(t *testing.T) { Response: HTTPResponse{ Code: 302, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Location"), - model.ArchivalMaybeBinaryString("https://x.example.com"), + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("https://x.example.com"), }, { - model.ArchivalMaybeBinaryString("Location"), - model.ArchivalMaybeBinaryString("https://y.example.com"), + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("https://y.example.com"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Server": "miniooni/0.1.0-dev", "Location": "https://x.example.com", }, diff --git a/internal/measurexlite/http.go b/internal/measurexlite/http.go index 8c196926ce..e595629308 100644 --- a/internal/measurexlite/http.go +++ b/internal/measurexlite/http.go @@ -51,7 +51,7 @@ func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, a ALPN: alpn, Failure: NewFailure(err), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: newHTTPRequestHeaderList(req), Headers: newHTTPRequestHeaderMap(req), @@ -61,7 +61,7 @@ func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, a URL: httpRequestURL(req), }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(body), + Body: model.ArchivalScrubbedMaybeBinaryString(body), BodyIsTruncated: httpResponseBodyIsTruncated(body, maxRespBodySize), Code: httpResponseStatusCode(resp), HeadersList: newHTTPResponseHeaderList(resp), @@ -95,7 +95,7 @@ func newHTTPRequestHeaderList(req *http.Request) []model.ArchivalHTTPHeader { // newHTTPRequestHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalMaybeBinaryString { +func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalScrubbedMaybeBinaryString { m := http.Header{} if req != nil { m = req.Header @@ -139,7 +139,7 @@ func newHTTPResponseHeaderList(resp *http.Response) (out []model.ArchivalHTTPHea // newHTTPResponseHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryString) { +func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalScrubbedMaybeBinaryString) { m := http.Header{} if resp != nil { m = resp.Header diff --git a/internal/measurexlite/http_test.go b/internal/measurexlite/http_test.go index c72abcde4d..ed31374170 100644 --- a/internal/measurexlite/http_test.go +++ b/internal/measurexlite/http_test.go @@ -58,21 +58,21 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Method: "", Tor: model.ArchivalHTTPTor{}, Transport: "", URL: "", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: []string{}, }, T0: 0, @@ -117,16 +117,16 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { return &s }(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("*/*"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "*/*", "User-Agent": "miniooni/0.1.0-dev", }, @@ -136,11 +136,11 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "http://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: []string{}, }, T0: 0.25, @@ -188,16 +188,16 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "h3", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("*/*"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "*/*", "User-Agent": "miniooni/0.1.0-dev", }, @@ -207,19 +207,19 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "https://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( testingx.HTTPBlockpage451, ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/html; charset=iso-8859-1"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=iso-8859-1"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/html; charset=iso-8859-1", "Server": "Apache", }, @@ -278,16 +278,16 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "h3", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("*/*"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0-dev"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "*/*", "User-Agent": "miniooni/0.1.0-dev", }, @@ -297,20 +297,20 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "https://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 302, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/html; charset=iso-8859-1"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=iso-8859-1"), }, { - model.ArchivalMaybeBinaryString("Location"), - model.ArchivalMaybeBinaryString("/v2/index.html"), + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("/v2/index.html"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/html; charset=iso-8859-1", "Location": "/v2/index.html", "Server": "Apache", diff --git a/internal/model/archival.go b/internal/model/archival.go index 514120ec00..474e2163c3 100644 --- a/internal/model/archival.go +++ b/internal/model/archival.go @@ -19,6 +19,8 @@ import ( "net/http" "sort" "unicode/utf8" + + "github.com/ooni/probe-cli/v3/internal/scrubber" ) // @@ -121,23 +123,27 @@ func (value *ArchivalBinaryData) UnmarshalJSON(raw []byte) error { return nil } -// ArchivalMaybeBinaryString is a possibly-binary string. When the string is valid UTF-8 +// ArchivalScrubbedMaybeBinaryString is a possibly-binary string. When the string is valid UTF-8 // we serialize it as itself. Otherwise, we use the binary data format defined by // https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata -type ArchivalMaybeBinaryString string +// +// As the name implies, the data contained by this type is scrubbed to remove IPv4 and IPv6 +// addresses and endpoints during JSON serialization, to make it less likely that OONI leaks +// IP addresses in textual or binary fields such as HTTP headers and bodies. +type ArchivalScrubbedMaybeBinaryString string var ( - _ json.Marshaler = ArchivalMaybeBinaryString("") - _ json.Unmarshaler = (func() *ArchivalMaybeBinaryString { return nil }()) + _ json.Marshaler = ArchivalScrubbedMaybeBinaryString("") + _ json.Unmarshaler = (func() *ArchivalScrubbedMaybeBinaryString { return nil }()) ) // MarshalJSON implements json.Marshaler. -func (value ArchivalMaybeBinaryString) MarshalJSON() ([]byte, error) { +func (value ArchivalScrubbedMaybeBinaryString) MarshalJSON() ([]byte, error) { // convert value to a string str := string(value) - // TODO(bassosimone): here is where we should scrub the string in the future - // once we have replaced AchivalMaybeBinaryData with ArchivalMaybeBinaryString + // make sure we get rid of IPv4 and IPv6 addresses and endpoints + str = scrubber.ScrubString(str) // if we can serialize as UTF-8 string, do that if utf8.ValidString(str) { @@ -149,11 +155,11 @@ func (value ArchivalMaybeBinaryString) MarshalJSON() ([]byte, error) { } // UnmarshalJSON implements json.Unmarshaler. -func (value *ArchivalMaybeBinaryString) UnmarshalJSON(rawData []byte) error { +func (value *ArchivalScrubbedMaybeBinaryString) UnmarshalJSON(rawData []byte) error { // first attempt to decode as a string var s string if err := json.Unmarshal(rawData, &s); err == nil { - *value = ArchivalMaybeBinaryString(s) + *value = ArchivalScrubbedMaybeBinaryString(s) return nil } @@ -162,7 +168,7 @@ func (value *ArchivalMaybeBinaryString) UnmarshalJSON(rawData []byte) error { if err := json.Unmarshal(rawData, &d); err != nil { return err } - *value = ArchivalMaybeBinaryString(d) + *value = ArchivalScrubbedMaybeBinaryString(d) return nil } @@ -173,6 +179,8 @@ func (value *ArchivalMaybeBinaryString) UnmarshalJSON(rawData []byte) error { // See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata // // Deprecated: do not use this type in new code. +// +// Removing this struct is TODO(https://github.com/ooni/probe/issues/2543). type ArchivalMaybeBinaryData struct { Value string } @@ -325,14 +333,14 @@ type ArchivalHTTPRequestResult struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPRequest struct { - Body ArchivalMaybeBinaryString `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryString `json:"headers"` - Method string `json:"method"` - Tor ArchivalHTTPTor `json:"tor"` - Transport string `json:"x_transport"` - URL string `json:"url"` + Body ArchivalScrubbedMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalScrubbedMaybeBinaryString `json:"headers"` + Method string `json:"method"` + Tor ArchivalHTTPTor `json:"tor"` + Transport string `json:"x_transport"` + URL string `json:"url"` } // ArchivalHTTPResponse contains an HTTP response. @@ -340,11 +348,11 @@ type ArchivalHTTPRequest struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPResponse struct { - Body ArchivalMaybeBinaryString `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - Code int64 `json:"code"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryString `json:"headers"` + Body ArchivalScrubbedMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + Code int64 `json:"code"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalScrubbedMaybeBinaryString `json:"headers"` // The following fields are not serialised but are useful to simplify // analysing the measurements in telegram, whatsapp, etc. @@ -370,8 +378,8 @@ func ArchivalNewHTTPHeadersList(source http.Header) (out []ArchivalHTTPHeader) { for _, key := range keys { for _, value := range source[key] { out = append(out, ArchivalHTTPHeader{ - ArchivalMaybeBinaryString(key), - ArchivalMaybeBinaryString(value), + ArchivalScrubbedMaybeBinaryString(key), + ArchivalScrubbedMaybeBinaryString(value), }) } } @@ -379,11 +387,11 @@ func ArchivalNewHTTPHeadersList(source http.Header) (out []ArchivalHTTPHeader) { } // ArchivalNewHTTPHeadersMap creates a map representation of HTTP headers -func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalMaybeBinaryString) { - out = make(map[string]ArchivalMaybeBinaryString) +func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalScrubbedMaybeBinaryString) { + out = make(map[string]ArchivalScrubbedMaybeBinaryString) for key, values := range header { for _, value := range values { - out[key] = ArchivalMaybeBinaryString(value) + out[key] = ArchivalScrubbedMaybeBinaryString(value) break // just the first header } } @@ -391,14 +399,14 @@ func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalMaybe } // ArchivalHTTPHeader is a single HTTP header. -type ArchivalHTTPHeader [2]ArchivalMaybeBinaryString +type ArchivalHTTPHeader [2]ArchivalScrubbedMaybeBinaryString // errCannotParseArchivalHTTPHeader indicates that we cannot parse an ArchivalHTTPHeader. var errCannotParseArchivalHTTPHeader = errors.New("invalid ArchivalHTTPHeader") // UnmarshalJSON implements json.Unmarshaler. func (ahh *ArchivalHTTPHeader) UnmarshalJSON(data []byte) error { - var helper []ArchivalMaybeBinaryString + var helper []ArchivalScrubbedMaybeBinaryString if err := json.Unmarshal(data, &helper); err != nil { return err } diff --git a/internal/model/archival_test.go b/internal/model/archival_test.go index 1979ed7024..6567ce4cfa 100644 --- a/internal/model/archival_test.go +++ b/internal/model/archival_test.go @@ -294,10 +294,10 @@ func TestArchivalBinaryData(t *testing.T) { }) } -func TestArchivalMaybeBinaryString(t *testing.T) { +func TestArchivalScrubbedMaybeBinaryString(t *testing.T) { t.Run("Supports assignment from a nil byte array", func(t *testing.T) { var data []byte = nil // explicit - casted := model.ArchivalMaybeBinaryString(data) + casted := model.ArchivalScrubbedMaybeBinaryString(data) if casted != "" { t.Fatal("unexpected value") } @@ -312,7 +312,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { name string // input is the possibly-binary input - input model.ArchivalMaybeBinaryString + input model.ArchivalScrubbedMaybeBinaryString // expectErr is the error we expect to see or nil expectErr error @@ -333,9 +333,14 @@ func TestArchivalMaybeBinaryString(t *testing.T) { expectData: []byte(`"Elliot"`), }, { name: "with value being a long binary string", - input: model.ArchivalMaybeBinaryString(archivalBinaryInput), + input: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), expectErr: nil, expectData: archivalEncodedBinaryInput, + }, { + name: "with string containing IP addresses and endpoints", + input: "a 130.192.91.211 b ::1 c [::1]:443 d 130.192.91.211:80", + expectErr: nil, + expectData: []byte(`"a [scrubbed] b [scrubbed] c [scrubbed] d [scrubbed]"`), }} for _, tc := range cases { @@ -386,85 +391,85 @@ func TestArchivalMaybeBinaryString(t *testing.T) { expectErr error // expectData is the data we expect - expectData model.ArchivalMaybeBinaryString + expectData model.ArchivalScrubbedMaybeBinaryString } cases := []testcase{{ name: "with nil input array", input: nil, expectErr: errors.New("unexpected end of JSON input"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with zero-length input array", input: []byte{}, expectErr: errors.New("unexpected end of JSON input"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with binary input that is not a complete JSON", input: []byte("{"), expectErr: errors.New("unexpected end of JSON input"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with ~random binary data as input", input: archivalBinaryInput, expectErr: errors.New("invalid character 'W' looking for beginning of value"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with valid JSON of the wrong type (array)", input: []byte("[]"), expectErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with valid JSON of the wrong type (number)", input: []byte("1.17"), expectErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with input being the liternal null", input: []byte(`null`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with empty JSON object", input: []byte("{}"), expectErr: errors.New("model: invalid binary data format: ''"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with correct data model but invalid format", input: []byte(`{"data":"","format":"antani"}`), expectErr: errors.New("model: invalid binary data format: 'antani'"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with correct data model and format but invalid base64 string", input: []byte(`{"data":"x","format":"base64"}`), expectErr: errors.New("illegal base64 data at input byte 0"), - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with correct data model and format but empty base64 string", input: []byte(`{"data":"","format":"base64"}`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString(""), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { name: "with the a string", input: []byte(`"Elliot"`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString("Elliot"), + expectData: model.ArchivalScrubbedMaybeBinaryString("Elliot"), }, { name: "with the encoding of a string", input: []byte(`{"data":"RWxsaW90","format":"base64"}`), expectErr: nil, - expectData: model.ArchivalMaybeBinaryString("Elliot"), + expectData: model.ArchivalScrubbedMaybeBinaryString("Elliot"), }, { name: "with the encoding of a complex binary string", input: archivalEncodedBinaryInput, expectErr: nil, - expectData: model.ArchivalMaybeBinaryString(archivalBinaryInput), + expectData: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }} for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { // unmarshal the raw input into an ArchivalBinaryData type - var abd model.ArchivalMaybeBinaryString + var abd model.ArchivalScrubbedMaybeBinaryString err := json.Unmarshal(tc.input, &abd) t.Log("got this error", err) @@ -503,7 +508,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { name string // input is the maybe-binary input - input model.ArchivalMaybeBinaryString + input model.ArchivalScrubbedMaybeBinaryString } cases := []testcase{{ @@ -514,7 +519,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { input: "Elliot", }, { name: "with value being a long binary string", - input: model.ArchivalMaybeBinaryString(archivalBinaryInput), + input: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }} for _, tc := range cases { @@ -531,7 +536,7 @@ func TestArchivalMaybeBinaryString(t *testing.T) { } // parse from JSON - var abc model.ArchivalMaybeBinaryString + var abc model.ArchivalScrubbedMaybeBinaryString if err := json.Unmarshal(output, &abc); err != nil { t.Fatal(err) } @@ -640,16 +645,16 @@ func TestArchivalHTTPHeader(t *testing.T) { }{{ name: "with string value", input: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/plain"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, want: []byte(`["Content-Type","text/plain"]`), wantErr: false, }, { name: "with binary value", input: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString(archivalBinaryInput), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }, want: []byte(`["Content-Type",` + string(archivalEncodedBinaryInput) + `]`), wantErr: false, @@ -677,98 +682,98 @@ func TestArchivalHTTPHeader(t *testing.T) { name: "with invalid input", input: []byte(`{}`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: errors.New("json: cannot unmarshal object into Go value of type []model.ArchivalMaybeBinaryString"), + wantErr: errors.New("json: cannot unmarshal object into Go value of type []model.ArchivalScrubbedMaybeBinaryString"), }, { name: "with zero items", input: []byte(`[]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 0"), }, { name: "with just one item", input: []byte(`["x"]`), - want: [2]model.ArchivalMaybeBinaryString{}, + want: [2]model.ArchivalScrubbedMaybeBinaryString{}, wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 1"), }, { name: "with three items", input: []byte(`["x","x","x"]`), - want: [2]model.ArchivalMaybeBinaryString{}, + want: [2]model.ArchivalScrubbedMaybeBinaryString{}, wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 3"), }, { name: "with first item not being a string", input: []byte(`[0,0]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), }, { name: "with both items being a string", input: []byte(`["x","y"]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("x"), - model.ArchivalMaybeBinaryString("y"), + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString("y"), }, wantErr: nil, }, { name: "with second item not being a map[string]interface{}", input: []byte(`["x",[]]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), }, { name: "with missing format key in second item", input: []byte(`["x",{}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("model: invalid binary data format: ''"), }, { name: "with format value not being base64", input: []byte(`["x",{"format":1}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal number into Go struct field archivalBinaryDataRepr.format of type string"), }, { name: "with missing data field", input: []byte(`["x",{"format":"base64"}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("x"), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: nil, }, { name: "with data not being a string", input: []byte(`["x",{"format":"base64","data":1}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("json: cannot unmarshal number into Go struct field archivalBinaryDataRepr.data of type []uint8"), }, { name: "with data not being base64", input: []byte(`["x",{"format":"base64","data":"xx"}]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString(""), - model.ArchivalMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, wantErr: errors.New("illegal base64 data at input byte 0"), }, { name: "with correctly encoded base64 data", input: []byte(`["x",` + string(archivalEncodedBinaryInput) + `]`), want: model.ArchivalHTTPHeader{ - model.ArchivalMaybeBinaryString("x"), - model.ArchivalMaybeBinaryString(archivalBinaryInput), + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }, wantErr: nil, }} @@ -1453,16 +1458,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1476,19 +1481,19 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "Bonsoir, Elliot!", ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", }, @@ -1515,16 +1520,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { return &s })(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1538,11 +1543,11 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: nil, }, T0: 0.4, @@ -1567,25 +1572,25 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }, { - model.ArchivalMaybeBinaryString("Antani"), - model.ArchivalMaybeBinaryString(string(archivalBinaryInput[:7])), + model.ArchivalScrubbedMaybeBinaryString("Antani"), + model.ArchivalScrubbedMaybeBinaryString(string(archivalBinaryInput[:7])), }, { - model.ArchivalMaybeBinaryString("Antani"), - model.ArchivalMaybeBinaryString(archivalBinaryInput[7:14]), + model.ArchivalScrubbedMaybeBinaryString("Antani"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[7:14]), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", - "Antani": model.ArchivalMaybeBinaryString(archivalBinaryInput[:7]), + "Antani": model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[:7]), }, Method: "GET", Tor: model.ArchivalHTTPTor{ @@ -1597,28 +1602,28 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( archivalBinaryInput[:77], ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }, { - model.ArchivalMaybeBinaryString("Mascetti"), - model.ArchivalMaybeBinaryString(archivalBinaryInput[14:21]), + model.ArchivalScrubbedMaybeBinaryString("Mascetti"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[14:21]), }, { - model.ArchivalMaybeBinaryString("Mascetti"), - model.ArchivalMaybeBinaryString(archivalBinaryInput[21:28]), + model.ArchivalScrubbedMaybeBinaryString("Mascetti"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[21:28]), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", - "Mascetti": model.ArchivalMaybeBinaryString(archivalEncodedBinaryInput[14:21]), + "Mascetti": model.ArchivalScrubbedMaybeBinaryString(archivalEncodedBinaryInput[14:21]), }, Locations: nil, }, @@ -1649,28 +1654,28 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }, { - model.ArchivalMaybeBinaryString("AntaniV4"), - model.ArchivalMaybeBinaryString("130.192.91.211"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV4"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211"), }, { - model.ArchivalMaybeBinaryString("AntaniV6"), - model.ArchivalMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV6"), + model.ArchivalScrubbedMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), }, { - model.ArchivalMaybeBinaryString("AntaniV4Epnt"), - model.ArchivalMaybeBinaryString("[130.192.91.211]:443"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV4Epnt"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211:443"), }, { - model.ArchivalMaybeBinaryString("AntaniV6Epnt"), - model.ArchivalMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), + model.ArchivalScrubbedMaybeBinaryString("AntaniV6Epnt"), + model.ArchivalScrubbedMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", "AntaniV4": "130.192.91.211", @@ -1688,31 +1693,31 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "Your address is 130.192.91.211 and 2606:2800:220:1:248:1893:25c8:1946 and you have endpoints [2606:2800:220:1:248:1893:25c8:1946]:5222 and 130.192.91.211:443. You're welcome.", ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }, { - model.ArchivalMaybeBinaryString("MascettiV4"), - model.ArchivalMaybeBinaryString("130.192.91.211"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV4"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211"), }, { - model.ArchivalMaybeBinaryString("MascettiV6"), - model.ArchivalMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV6"), + model.ArchivalScrubbedMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), }, { - model.ArchivalMaybeBinaryString("MascettiV4Epnt"), - model.ArchivalMaybeBinaryString("[130.192.91.211]:443"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV4Epnt"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211:443"), }, { - model.ArchivalMaybeBinaryString("MascettiV6Epnt"), - model.ArchivalMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), + model.ArchivalScrubbedMaybeBinaryString("MascettiV6Epnt"), + model.ArchivalScrubbedMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", "MascettiV4": "130.192.91.211", @@ -1728,7 +1733,7 @@ func TestArchivalHTTPRequestResult(t *testing.T) { TransactionID: 5, }, expectErr: nil, - expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"],["AntaniV4","130.192.91.211"],["AntaniV6","2606:2800:220:1:248:1893:25c8:1946"],["AntaniV4Epnt","[130.192.91.211]:443"],["AntaniV6Epnt","[2606:2800:220:1:248:1893:25c8:1946]:5222"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","AntaniV4":"130.192.91.211","AntaniV4Epnt":"130.192.91.211:443","AntaniV6":"2606:2800:220:1:248:1893:25c8:1946","AntaniV6Epnt":"[2606:2800:220:1:248:1893:25c8:1946]:5222","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"\u003cHTML\u003e\u003cBODY\u003eYour address is 130.192.91.211 and 2606:2800:220:1:248:1893:25c8:1946 and you have endpoints [2606:2800:220:1:248:1893:25c8:1946]:5222 and 130.192.91.211:443. You're welcome.\u003c/BODY\u003e\u003c/HTML\u003e","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"],["MascettiV4","130.192.91.211"],["MascettiV6","2606:2800:220:1:248:1893:25c8:1946"],["MascettiV4Epnt","[130.192.91.211]:443"],["MascettiV6Epnt","[2606:2800:220:1:248:1893:25c8:1946]:5222"]],"headers":{"Age":"131833","MascettiV4":"130.192.91.211","MascettiV4Epnt":"130.192.91.211:443","MascettiV6":"2606:2800:220:1:248:1893:25c8:1946","MascettiV6Epnt":"[2606:2800:220:1:248:1893:25c8:1946]:5222","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), + expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"],["AntaniV4","[scrubbed]"],["AntaniV6","[scrubbed]"],["AntaniV4Epnt","[scrubbed]"],["AntaniV6Epnt","[scrubbed]"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","AntaniV4":"[scrubbed]","AntaniV4Epnt":"[scrubbed]","AntaniV6":"[scrubbed]","AntaniV6Epnt":"[scrubbed]","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"\u003cHTML\u003e\u003cBODY\u003eYour address is [scrubbed] and [scrubbed] and you have endpoints [scrubbed] and [scrubbed]. You're welcome.\u003c/BODY\u003e\u003c/HTML\u003e","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"],["MascettiV4","[scrubbed]"],["MascettiV6","[scrubbed]"],["MascettiV4Epnt","[scrubbed]"],["MascettiV6Epnt","[scrubbed]"]],"headers":{"Age":"131833","MascettiV4":"[scrubbed]","MascettiV4Epnt":"[scrubbed]","MascettiV6":"[scrubbed]","MascettiV6Epnt":"[scrubbed]","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), }, } @@ -1793,16 +1798,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { ALPN: "h2", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1816,19 +1821,19 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString( + Body: model.ArchivalScrubbedMaybeBinaryString( "Bonsoir, Elliot!", ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Age"), - model.ArchivalMaybeBinaryString("131833"), + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), }, { - model.ArchivalMaybeBinaryString("Server"), - model.ArchivalMaybeBinaryString("Apache"), + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Age": "131833", "Server": "Apache", }, @@ -1852,16 +1857,16 @@ func TestArchivalHTTPRequestResult(t *testing.T) { return &s })(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Accept"), - model.ArchivalMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }}, - Headers: map[string]model.ArchivalMaybeBinaryString{ + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "User-Agent": "miniooni/0.1.0", }, @@ -1875,11 +1880,11 @@ func TestArchivalHTTPRequestResult(t *testing.T) { URL: "https://www.example.com/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryString(""), + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryString{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: nil, }, T0: 0.4, @@ -2125,20 +2130,20 @@ func TestArchivalNewHTTPHeadersList(t *testing.T) { "User-Agent": {"miniooni/0.1.0"}, }, expect: []model.ArchivalHTTPHeader{{ - model.ArchivalMaybeBinaryString("Content-Type"), - model.ArchivalMaybeBinaryString("text/html; charset=utf-8"), + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=utf-8"), }, { - model.ArchivalMaybeBinaryString("User-Agent"), - model.ArchivalMaybeBinaryString("miniooni/0.1.0"), + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), }, { - model.ArchivalMaybeBinaryString("Via"), - model.ArchivalMaybeBinaryString("a"), + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("a"), }, { - model.ArchivalMaybeBinaryString("Via"), - model.ArchivalMaybeBinaryString("b"), + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("b"), }, { - model.ArchivalMaybeBinaryString("Via"), - model.ArchivalMaybeBinaryString("c"), + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("c"), }}, }} @@ -2158,17 +2163,17 @@ func TestArchivalNewHTTPHeadersMap(t *testing.T) { type testcase struct { name string input http.Header - expect map[string]model.ArchivalMaybeBinaryString + expect map[string]model.ArchivalScrubbedMaybeBinaryString } cases := []testcase{{ name: "with nil input", input: nil, - expect: map[string]model.ArchivalMaybeBinaryString{}, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, { name: "with empty input", input: map[string][]string{}, - expect: map[string]model.ArchivalMaybeBinaryString{}, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, { name: "common case", input: map[string][]string{ @@ -2176,7 +2181,7 @@ func TestArchivalNewHTTPHeadersMap(t *testing.T) { "Via": {"a", "b", "c"}, "User-Agent": {"miniooni/0.1.0"}, }, - expect: map[string]model.ArchivalMaybeBinaryString{ + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{ "Content-Type": "text/html; charset=utf-8", "Via": "a", "User-Agent": "miniooni/0.1.0",