From 6d74d13e4bddf5ad6cf33dd7917660b77c9d6cb1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 13:36:22 +0200 Subject: [PATCH 01/10] build-cli.sh: linux: build static binary 1. ask for static linking 2. use netgo Just tested with Docker using Ubuntu 12.04 Closes https://github.com/ooni/probe-engine/issues/815 --- build-cli.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-cli.sh b/build-cli.sh index 6a274189..2be4202a 100755 --- a/build-cli.sh +++ b/build-cli.sh @@ -6,7 +6,7 @@ case $1 in go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./cmd/miniooni;; linux) export GOOS=linux GOARCH=amd64 - go build -o ./CLI/linux/amd64 -ldflags="-s -w" ./cmd/miniooni;; + go build -o ./CLI/linux/amd64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/miniooni;; windows) export GOOS=windows GOARCH=amd64 go build -o ./CLI/windows/amd64 -ldflags="-s -w" ./cmd/miniooni;; From 6b8da77b1a78b78149fc6c36b404c3ae2519d437 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 13:43:55 +0200 Subject: [PATCH 02/10] oonimkall: remove misleading comment Closes https://github.com/ooni/probe-engine/issues/816 --- oonimkall/runner.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/oonimkall/runner.go b/oonimkall/runner.go index f3985975..0df2507f 100644 --- a/oonimkall/runner.go +++ b/oonimkall/runner.go @@ -292,8 +292,6 @@ func (r *runner) Run(ctx context.Context) { endEvent.DownloadedKB = experiment.KibiBytesReceived() endEvent.UploadedKB = experiment.KibiBytesSent() }() - // TODO(bassosimone): here we should recognise the NoEndpoint - // setting used by the WhatsApp experiment. if !r.settings.Options.NoCollector { logger.Info("Opening report... please, be patient") if err := experiment.OpenReport(); err != nil { From ac53fe569b54c93757daa7a49dbb51c1364fc9e9 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 13:52:55 +0200 Subject: [PATCH 03/10] urlgetter: rename MaybeUserAgent Closes https://github.com/ooni/probe-engine/issues/817 --- experiment/urlgetter/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/experiment/urlgetter/runner.go b/experiment/urlgetter/runner.go index 2b2f31d5..fa005201 100644 --- a/experiment/urlgetter/runner.go +++ b/experiment/urlgetter/runner.go @@ -52,9 +52,9 @@ func (r Runner) Run(ctx context.Context) error { } } -// MaybeRandomUserAgent returns ua if ua is not empty. Otherwise it +// MaybeUserAgent returns ua if ua is not empty. Otherwise it // returns httpheader.RandomUserAgent(). -func MaybeRandomUserAgent(ua string) string { +func MaybeUserAgent(ua string) string { if ua == "" { ua = httpheader.UserAgent() } @@ -68,7 +68,7 @@ func (r Runner) httpGet(ctx context.Context, url string) error { req = req.WithContext(ctx) req.Header.Set("Accept", httpheader.Accept()) req.Header.Set("Accept-Language", httpheader.AcceptLanguage()) - req.Header.Set("User-Agent", MaybeRandomUserAgent(r.Config.UserAgent)) + req.Header.Set("User-Agent", MaybeUserAgent(r.Config.UserAgent)) if r.Config.HTTPHost != "" { req.Host = r.Config.HTTPHost } From 1181de5e5d9db234ef02eb0bcb44d2c76aeb6ad5 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 13:54:11 +0200 Subject: [PATCH 04/10] whatsapp: re-enable previously disabled test A recent build of the stable branch shows that there is no failure now: https://github.com/ooni/probe-engine/actions/runs/176959541 This means we can close https://github.com/ooni/probe-engine/issues/799 --- experiment/whatsapp/whatsapp_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/experiment/whatsapp/whatsapp_test.go b/experiment/whatsapp/whatsapp_test.go index 3e256122..107e6fd4 100644 --- a/experiment/whatsapp/whatsapp_test.go +++ b/experiment/whatsapp/whatsapp_test.go @@ -28,7 +28,6 @@ func TestNewExperimentMeasurer(t *testing.T) { } func TestIntegrationSuccess(t *testing.T) { - t.Skip() // See https://github.com/ooni/probe-engine/issues/799 measurer := whatsapp.NewExperimentMeasurer(whatsapp.Config{}) ctx := context.Background() sess := &mockable.ExperimentSession{MockableLogger: log.Log} From 97f9dcc93d71de7d837ab85ac296ebcf6dfa142a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 14:02:32 +0200 Subject: [PATCH 05/10] hhfm, hirl: fill the backend field Closes https://github.com/ooni/probe-engine/issues/777 --- experiment/hhfm/hhfm.go | 3 +++ experiment/hirl/hirl.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/experiment/hhfm/hhfm.go b/experiment/hhfm/hhfm.go index cade9ed7..74c2b6c5 100644 --- a/experiment/hhfm/hhfm.go +++ b/experiment/hhfm/hhfm.go @@ -116,6 +116,9 @@ func (m Measurer) Run( if helper.Type != "legacy" { return ErrInvalidHelperType } + measurement.TestHelpers = map[string]interface{}{ + "backend": helper.Address, + } // prepare request req, err := http.NewRequest("GeT", helper.Address, nil) if err != nil { diff --git a/experiment/hirl/hirl.go b/experiment/hirl/hirl.go index 9b6491da..a996b9c1 100644 --- a/experiment/hirl/hirl.go +++ b/experiment/hirl/hirl.go @@ -93,6 +93,9 @@ func (m Measurer) Run( if helper.Type != "legacy" { return ErrInvalidHelperType } + measurement.TestHelpers = map[string]interface{}{ + "backend": helper.Address, + } out := make(chan MethodResult) for _, method := range m.Methods { callbacks.OnProgress(0.0, fmt.Sprintf("%s...", method.Name())) From b10b473b16d0d42da394298d331ba8fe53cff9ff Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 14:14:11 +0200 Subject: [PATCH 06/10] oonimkall: add API to generate a UUID4 This implementation panics if we cannot generate a UUID4. I think that is fine in this case. We can't be OONI without a working CSRNG. Closes https://github.com/ooni/probe-engine/issues/738 --- go.mod | 1 + go.sum | 1 + oonimkall/uuid.go | 8 ++++++++ oonimkall/uuid_test.go | 13 +++++++++++++ 4 files changed, 23 insertions(+) create mode 100644 oonimkall/uuid.go create mode 100644 oonimkall/uuid_test.go diff --git a/go.mod b/go.mod index d932699a..447f6c45 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439 // indirect github.com/google/go-cmp v0.5.0 github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 // indirect + github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.2 github.com/grafov/m3u8 v0.0.0-20171211212457-6ab8f28ed427 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect diff --git a/go.sum b/go.sum index 0735ff1a..dc139dd8 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 h1:OL2d27ueTKnlQJoqLW2fc9pWYulFnJYLWzomGV7HqZo= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4/go.mod h1:Pw1H1OjSNHiqeuxAduB1BKYXIwFtsyrY47nEqSgEiCM= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= diff --git a/oonimkall/uuid.go b/oonimkall/uuid.go new file mode 100644 index 00000000..70578e60 --- /dev/null +++ b/oonimkall/uuid.go @@ -0,0 +1,8 @@ +package oonimkall + +import "github.com/google/uuid" + +// NewUUID4 generates a new UUID4. +func NewUUID4() string { + return uuid.Must(uuid.NewRandom()).String() +} diff --git a/oonimkall/uuid_test.go b/oonimkall/uuid_test.go new file mode 100644 index 00000000..5b83e1e3 --- /dev/null +++ b/oonimkall/uuid_test.go @@ -0,0 +1,13 @@ +package oonimkall_test + +import ( + "testing" + + "github.com/ooni/probe-engine/oonimkall" +) + +func TestNewUUID4(t *testing.T) { + if out := oonimkall.NewUUID4(); len(out) != 36 { + t.Fatal("not the expected output") + } +} From 5a965adb5dd56efbbaada0605c3c5b6feb6d2f41 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 14:25:10 +0200 Subject: [PATCH 07/10] Upgrade direct and indirect dependencies --- go.mod | 15 +++++++-------- go.sum | 37 ++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 447f6c45..13eeb776 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,12 @@ require ( github.com/Psiphon-Labs/goarista v0.0.0-20160825065156-d002785f4c67 // indirect github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 // indirect github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 // indirect - github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible + github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab // indirect - github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797 // indirect + github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect - github.com/apex/log v1.4.0 - github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a // indirect + github.com/apex/log v1.6.0 + github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 // indirect github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 // indirect @@ -38,10 +38,9 @@ require ( github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a // indirect - github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b + github.com/miekg/dns v1.1.30 github.com/montanaflynn/stats v0.6.3 github.com/oschwald/geoip2-golang v1.4.0 - github.com/oschwald/maxminddb-golang v1.7.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 github.com/pion/stun v0.3.5 @@ -59,6 +58,6 @@ require ( go.uber.org/atomic v1.3.3-0.20180806045314-ca680462431f // indirect go.uber.org/multierr v1.1.1-0.20180122172545-ddea229ff1df // indirect go.uber.org/zap v1.9.2-0.20180814183419-67bc79d13d15 // indirect - golang.org/x/net v0.0.0-20200625001655-4c5254603344 - golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 // indirect ) diff --git a/go.sum b/go.sum index dc139dd8..3123ba40 100644 --- a/go.sum +++ b/go.sum @@ -21,12 +21,12 @@ github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 h1:VmnMMMheFX github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464/go.mod h1:Pe5BqN2DdIdChorAXl6bDaQd/wghpCleJfid2NoSli0= github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 h1:te4lDZfA3tFwaheo+h/GZYGiLGJvm7Dcq2YkFh13QmE= github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742/go.mod h1:3mBCrUrPxFCKAhG0ZdEfiU7QU6zl2+gr1HUk1sKYdjI= -github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible h1:+o+Bx2k8rk1PjekH4zUx65TcIinj7611JegDAFfuAvo= -github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible/go.mod h1:VcNEtiQ0z2sCGJf16ZGcpwCas5+r9rt+P20r6LlJ06U= +github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible h1:c76E9yKZiStloyBp1lNFFuGmjf73Kldsk3jpgahE7Mc= +github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible/go.mod h1:VcNEtiQ0z2sCGJf16ZGcpwCas5+r9rt+P20r6LlJ06U= github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab h1:LYp5/y2XR38yyDtNyEZBt0TtFaWgrYMNu3hEcN+om6c= github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab/go.mod h1:I0Z7XA8KzHZl2MzwqEbZJhBQOHT8ajGUQ5+SWf5KHw0= -github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797 h1:8jTZ+3BhLiYm3QJ4qH2l5Oeb8Vft5eMVG/cYk4uoi4s= -github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE= +github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f h1:DZpr9KUNwaL+OjdT7JR+bkr7xXkvikTSMdpt6L1XX6Y= +github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= @@ -37,15 +37,15 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/apex/log v1.4.0 h1:jYWeNt9kWJOf1ifht8UjsCQ00eiPnFrUzCBCiiJMw/g= -github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE= -github.com/apex/logs v0.0.7/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/apex/log v1.6.0 h1:Y50wF1PBIIexIgTm0/7G6gcLitkO5jHK5Mb6wcMY0UI= +github.com/apex/log v1.6.0/go.mod h1:x7s+P9VtvFBXge9Vbn+8TrqKmuzmD35TTkeBHul8UtY= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= -github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a h1:SLrT2O2v0QGaH1JSte1vJ2HtreWZa0MjdGSfXp25qlc= -github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE= +github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 h1:VbwKXgO1O1JSbI8o3PQqlC/KTem5t3YD7LqvfBT+0Gk= +github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE= github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f h1:SaJ6yqg936TshyeFZqQE+N+9hYkIeL9AMr7S4voCl10= github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= @@ -187,8 +187,8 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b h1:HOmliqRWsjtpSxWaulwlgzShQQCiI0yIaoxJoSDcxao= -github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo= +github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -212,8 +212,6 @@ github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9Pvr github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng= github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls= github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w= -github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc= -github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA= @@ -285,6 +283,9 @@ github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEv github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.0.1/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= @@ -341,8 +342,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -371,8 +372,8 @@ golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c h1:jceGD5YNJGgGMkJz79agzOln1 golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -428,5 +429,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From ab6e3bb2d58e588d74cb7a6d0c2f276bbec67159 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 14:27:14 +0200 Subject: [PATCH 08/10] useragent.go: update the user-agent we use for measuring Part of https://github.com/ooni/probe-engine/issues/757 --- internal/httpheader/useragent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/httpheader/useragent.go b/internal/httpheader/useragent.go index 5f9f5709..3649e674 100644 --- a/internal/httpheader/useragent.go +++ b/internal/httpheader/useragent.go @@ -3,7 +3,7 @@ package httpheader // UserAgent returns the User-Agent header used for measuring. func UserAgent() string { - // 14.9% as of Jun 25, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ - const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" + // 18.2% as of Jul 21, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ + const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" return ua } From 20fbe646772eacee6ad721512026b22a36edb1d7 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 14:28:07 +0200 Subject: [PATCH 09/10] version/version.go: we now pretend to be 0.15.0 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 3618469f..deba3b46 100644 --- a/version/version.go +++ b/version/version.go @@ -2,4 +2,4 @@ package version // Version is the version of the engine -const Version = "0.14.0" +const Version = "0.15.0" From bdea43c5fd0c58f06312562d4f18382cefe8bce6 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 21 Jul 2020 14:28:38 +0200 Subject: [PATCH 10/10] resources: upgrade assets to 20200721121920 --- internal/resources/assets.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/resources/assets.go b/internal/resources/assets.go index 86df4088..0f21b215 100644 --- a/internal/resources/assets.go +++ b/internal/resources/assets.go @@ -2,7 +2,7 @@ package resources const ( // Version contains the assets version. - Version = 20200619115947 + Version = 20200721121920 // ASNDatabaseName is the ASN-DB file name ASNDatabaseName = "asn.mmdb" @@ -33,18 +33,18 @@ type ResourceInfo struct { // All contains info on all known assets. var All = map[string]ResourceInfo{ "asn.mmdb": { - URLPath: "/ooni/probe-assets/releases/download/20200619115947/asn.mmdb.gz", - GzSHA256: "2bc9e7d0e445d3f93edad094ef9060288850f34652f7592bdedabb5cf74a2065", - SHA256: "d71f0de2fdf17c809e5f60e9b69f874acbe9cde108298ecaedfab16d166cf878", + URLPath: "/ooni/probe-assets/releases/download/20200721121920/asn.mmdb.gz", + GzSHA256: "e0f6adfbbbb565a04b97d9c78d77023e22f8a30477d36acab55169911cb6ba51", + SHA256: "600222c58a464c4cece3096b0b553c0db674276e81e5cd84e76b18acef1c2fce", }, "ca-bundle.pem": { - URLPath: "/ooni/probe-assets/releases/download/20200619115947/ca-bundle.pem.gz", - GzSHA256: "08070cbe24c8895d18bb20ccd746ff7409f1947094a1a47aa59993f588474485", - SHA256: "adf770dfd574a0d6026bfaa270cb6879b063957177a991d453ff1d302c02081f", + URLPath: "/ooni/probe-assets/releases/download/20200721121920/ca-bundle.pem.gz", + GzSHA256: "96de2f6469ce24c1909c82704df519566ec9ecddb7fd8b6ae635dba2a55c0e8c", + SHA256: "726889705b00f736200ed7999f7a50021b8735d53228d679c4e6665aa3b44987", }, "country.mmdb": { - URLPath: "/ooni/probe-assets/releases/download/20200619115947/country.mmdb.gz", - GzSHA256: "a3b13c78c149da4fa8bd63bbb0c25ead9737c1da3bf2e3199f2bca86d73c01b9", - SHA256: "83f369ddcb560862996848b600ce1e5353659dcb9424c5e9dd7f6d980fc56a60", + URLPath: "/ooni/probe-assets/releases/download/20200721121920/country.mmdb.gz", + GzSHA256: "5f23c9155ce6e786c2c94d154d67411f33b39d67f1206fee8d6c61b829acdad5", + SHA256: "cffb80632d05ea76188bacaf6c085bb41540b283b7189b5f5fed9ae7173e7967", }, }