Merge pull request #818 from ooni/prerelease

Prepare the master branch for 0.15.0 release

We will be branching for release off 27b280f, because we don't want to ship the new Web Connectivity implementation until we have done some more in depth QA of the new implementation. Yet, we want to ensure that master and the branch we will be using will as much as possible have a compatible set of changes, to reduce future pain. For this reason, this diff applies most of the changes required in the release branch to master. This work is part of #757.

build-cli.sh

@@ -6,7 +6,7 @@ case $1 in
     go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./cmd/miniooni;;
   linux)
     export GOOS=linux GOARCH=amd64
-    go build -o ./CLI/linux/amd64 -ldflags="-s -w" ./cmd/miniooni;;
+    go build -o ./CLI/linux/amd64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/miniooni;;
   windows)
     export GOOS=windows GOARCH=amd64
     go build -o ./CLI/windows/amd64 -ldflags="-s -w" ./cmd/miniooni;;

experiment/hhfm/hhfm.go

@@ -116,6 +116,9 @@ func (m Measurer) Run(
 	if helper.Type != "legacy" {
 		return ErrInvalidHelperType
 	}
+	measurement.TestHelpers = map[string]interface{}{
+		"backend": helper.Address,
+	}
 	// prepare request
 	req, err := http.NewRequest("GeT", helper.Address, nil)
 	if err != nil {

experiment/hirl/hirl.go

@@ -93,6 +93,9 @@ func (m Measurer) Run(
 	if helper.Type != "legacy" {
 		return ErrInvalidHelperType
 	}
+	measurement.TestHelpers = map[string]interface{}{
+		"backend": helper.Address,
+	}
 	out := make(chan MethodResult)
 	for _, method := range m.Methods {
 		callbacks.OnProgress(0.0, fmt.Sprintf("%s...", method.Name()))

experiment/urlgetter/runner.go

@@ -52,9 +52,9 @@ func (r Runner) Run(ctx context.Context) error {
 	}
 }
 
-// MaybeRandomUserAgent returns ua if ua is not empty. Otherwise it
+// MaybeUserAgent returns ua if ua is not empty. Otherwise it
 // returns httpheader.RandomUserAgent().
-func MaybeRandomUserAgent(ua string) string {
+func MaybeUserAgent(ua string) string {
 	if ua == "" {
 		ua = httpheader.UserAgent()
 	}
@@ -68,7 +68,7 @@ func (r Runner) httpGet(ctx context.Context, url string) error {
 	req = req.WithContext(ctx)
 	req.Header.Set("Accept", httpheader.Accept())
 	req.Header.Set("Accept-Language", httpheader.AcceptLanguage())
-	req.Header.Set("User-Agent", MaybeRandomUserAgent(r.Config.UserAgent))
+	req.Header.Set("User-Agent", MaybeUserAgent(r.Config.UserAgent))
 	if r.Config.HTTPHost != "" {
 		req.Host = r.Config.HTTPHost
 	}

experiment/whatsapp/whatsapp_test.go

@@ -28,7 +28,6 @@ func TestNewExperimentMeasurer(t *testing.T) {
 }
 
 func TestIntegrationSuccess(t *testing.T) {
-	t.Skip() // See https://github.com/ooni/probe-engine/issues/799
 	measurer := whatsapp.NewExperimentMeasurer(whatsapp.Config{})
 	ctx := context.Background()
 	sess := &mockable.ExperimentSession{MockableLogger: log.Log}

go.mod

@@ -11,12 +11,12 @@ require (
 	github.com/Psiphon-Labs/goarista v0.0.0-20160825065156-d002785f4c67 // indirect
 	github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 // indirect
 	github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 // indirect
-	github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible
+	github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible
 	github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab // indirect
-	github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797 // indirect
+	github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
-	github.com/apex/log v1.4.0
-	github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a // indirect
+	github.com/apex/log v1.6.0
+	github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 // indirect
 	github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f // indirect
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
 	github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 // indirect
@@ -31,16 +31,16 @@ require (
 	github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439 // indirect
 	github.com/google/go-cmp v0.5.0
 	github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 // indirect
+	github.com/google/uuid v1.1.1
 	github.com/gorilla/websocket v1.4.2
 	github.com/grafov/m3u8 v0.0.0-20171211212457-6ab8f28ed427 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
 	github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect
 	github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a // indirect
-	github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b
+	github.com/miekg/dns v1.1.30
 	github.com/montanaflynn/stats v0.6.3
 	github.com/oschwald/geoip2-golang v1.4.0
-	github.com/oschwald/maxminddb-golang v1.7.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
 	github.com/pion/stun v0.3.5
@@ -58,6 +58,6 @@ require (
 	go.uber.org/atomic v1.3.3-0.20180806045314-ca680462431f // indirect
 	go.uber.org/multierr v1.1.1-0.20180122172545-ddea229ff1df // indirect
 	go.uber.org/zap v1.9.2-0.20180814183419-67bc79d13d15 // indirect
-	golang.org/x/net v0.0.0-20200625001655-4c5254603344
-	golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 // indirect
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
+	golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 // indirect
 )

go.sum

@@ -21,12 +21,12 @@ github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 h1:VmnMMMheFX
 github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464/go.mod h1:Pe5BqN2DdIdChorAXl6bDaQd/wghpCleJfid2NoSli0=
 github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 h1:te4lDZfA3tFwaheo+h/GZYGiLGJvm7Dcq2YkFh13QmE=
 github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742/go.mod h1:3mBCrUrPxFCKAhG0ZdEfiU7QU6zl2+gr1HUk1sKYdjI=
-github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible h1:+o+Bx2k8rk1PjekH4zUx65TcIinj7611JegDAFfuAvo=
-github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible/go.mod h1:VcNEtiQ0z2sCGJf16ZGcpwCas5+r9rt+P20r6LlJ06U=
+github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible h1:c76E9yKZiStloyBp1lNFFuGmjf73Kldsk3jpgahE7Mc=
+github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible/go.mod h1:VcNEtiQ0z2sCGJf16ZGcpwCas5+r9rt+P20r6LlJ06U=
 github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab h1:LYp5/y2XR38yyDtNyEZBt0TtFaWgrYMNu3hEcN+om6c=
 github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab/go.mod h1:I0Z7XA8KzHZl2MzwqEbZJhBQOHT8ajGUQ5+SWf5KHw0=
-github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797 h1:8jTZ+3BhLiYm3QJ4qH2l5Oeb8Vft5eMVG/cYk4uoi4s=
-github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE=
+github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f h1:DZpr9KUNwaL+OjdT7JR+bkr7xXkvikTSMdpt6L1XX6Y=
+github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE=
 github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI=
@@ -37,15 +37,15 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/apex/log v1.4.0 h1:jYWeNt9kWJOf1ifht8UjsCQ00eiPnFrUzCBCiiJMw/g=
-github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE=
-github.com/apex/logs v0.0.7/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
+github.com/apex/log v1.6.0 h1:Y50wF1PBIIexIgTm0/7G6gcLitkO5jHK5Mb6wcMY0UI=
+github.com/apex/log v1.6.0/go.mod h1:x7s+P9VtvFBXge9Vbn+8TrqKmuzmD35TTkeBHul8UtY=
+github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
 github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
 github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
 github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks=
 github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA=
-github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a h1:SLrT2O2v0QGaH1JSte1vJ2HtreWZa0MjdGSfXp25qlc=
-github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE=
+github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 h1:VbwKXgO1O1JSbI8o3PQqlC/KTem5t3YD7LqvfBT+0Gk=
+github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE=
 github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc=
 github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f h1:SaJ6yqg936TshyeFZqQE+N+9hYkIeL9AMr7S4voCl10=
 github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU=
@@ -130,6 +130,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 h1:OL2d27ueTKnlQJoqLW2fc9pWYulFnJYLWzomGV7HqZo=
 github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4/go.mod h1:Pw1H1OjSNHiqeuxAduB1BKYXIwFtsyrY47nEqSgEiCM=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -186,8 +187,8 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b h1:HOmliqRWsjtpSxWaulwlgzShQQCiI0yIaoxJoSDcxao=
-github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
+github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -211,8 +212,6 @@ github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9Pvr
 github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
 github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls=
 github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
-github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc=
-github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA=
@@ -284,6 +283,9 @@ github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEv
 github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-buffer v1.0.1/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
@@ -340,8 +342,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -370,8 +372,8 @@ golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c h1:jceGD5YNJGgGMkJz79agzOln1
 golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8=
+golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -427,5 +429,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/httpheader/useragent.go

@@ -3,7 +3,7 @@ package httpheader
 
 // UserAgent returns the User-Agent header used for measuring.
 func UserAgent() string {
-	// 14.9% as of Jun 25, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
-	const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
+	// 18.2% as of Jul 21, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
+	const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
 	return ua
 }

internal/resources/assets.go

@@ -2,7 +2,7 @@ package resources
 
 const (
 	// Version contains the assets version.
-	Version = 20200619115947
+	Version = 20200721121920
 
 	// ASNDatabaseName is the ASN-DB file name
 	ASNDatabaseName = "asn.mmdb"
@@ -33,18 +33,18 @@ type ResourceInfo struct {
 // All contains info on all known assets.
 var All = map[string]ResourceInfo{
 	"asn.mmdb": {
-		URLPath:  "/ooni/probe-assets/releases/download/20200619115947/asn.mmdb.gz",
-		GzSHA256: "2bc9e7d0e445d3f93edad094ef9060288850f34652f7592bdedabb5cf74a2065",
-		SHA256:   "d71f0de2fdf17c809e5f60e9b69f874acbe9cde108298ecaedfab16d166cf878",
+		URLPath:  "/ooni/probe-assets/releases/download/20200721121920/asn.mmdb.gz",
+		GzSHA256: "e0f6adfbbbb565a04b97d9c78d77023e22f8a30477d36acab55169911cb6ba51",
+		SHA256:   "600222c58a464c4cece3096b0b553c0db674276e81e5cd84e76b18acef1c2fce",
 	},
 	"ca-bundle.pem": {
-		URLPath:  "/ooni/probe-assets/releases/download/20200619115947/ca-bundle.pem.gz",
-		GzSHA256: "08070cbe24c8895d18bb20ccd746ff7409f1947094a1a47aa59993f588474485",
-		SHA256:   "adf770dfd574a0d6026bfaa270cb6879b063957177a991d453ff1d302c02081f",
+		URLPath:  "/ooni/probe-assets/releases/download/20200721121920/ca-bundle.pem.gz",
+		GzSHA256: "96de2f6469ce24c1909c82704df519566ec9ecddb7fd8b6ae635dba2a55c0e8c",
+		SHA256:   "726889705b00f736200ed7999f7a50021b8735d53228d679c4e6665aa3b44987",
 	},
 	"country.mmdb": {
-		URLPath:  "/ooni/probe-assets/releases/download/20200619115947/country.mmdb.gz",
-		GzSHA256: "a3b13c78c149da4fa8bd63bbb0c25ead9737c1da3bf2e3199f2bca86d73c01b9",
-		SHA256:   "83f369ddcb560862996848b600ce1e5353659dcb9424c5e9dd7f6d980fc56a60",
+		URLPath:  "/ooni/probe-assets/releases/download/20200721121920/country.mmdb.gz",
+		GzSHA256: "5f23c9155ce6e786c2c94d154d67411f33b39d67f1206fee8d6c61b829acdad5",
+		SHA256:   "cffb80632d05ea76188bacaf6c085bb41540b283b7189b5f5fed9ae7173e7967",
 	},
 }

oonimkall/runner.go

@@ -292,8 +292,6 @@ func (r *runner) Run(ctx context.Context) {
 		endEvent.DownloadedKB = experiment.KibiBytesReceived()
 		endEvent.UploadedKB = experiment.KibiBytesSent()
 	}()
-	// TODO(bassosimone): here we should recognise the NoEndpoint
-	// setting used by the WhatsApp experiment.
 	if !r.settings.Options.NoCollector {
 		logger.Info("Opening report... please, be patient")
 		if err := experiment.OpenReport(); err != nil {

oonimkall/uuid.go

@@ -0,0 +1,8 @@
+package oonimkall
+
+import "github.com/google/uuid"
+
+// NewUUID4 generates a new UUID4.
+func NewUUID4() string {
+	return uuid.Must(uuid.NewRandom()).String()
+}

oonimkall/uuid_test.go

@@ -0,0 +1,13 @@
+package oonimkall_test
+
+import (
+	"testing"
+
+	"github.com/ooni/probe-engine/oonimkall"
+)
+
+func TestNewUUID4(t *testing.T) {
+	if out := oonimkall.NewUUID4(); len(out) != 36 {
+		t.Fatal("not the expected output")
+	}
+}

version/version.go

@@ -2,4 +2,4 @@
 package version
 
 // Version is the version of the engine
-const Version = "0.14.0"
+const Version = "0.15.0"