Make sure we test the right versions and the testing infra (#26)

* fix: run MK on macOS w/ testing infra only MK is increasingly less important to us. But we know that we have people running versions of MK compiled by themselves. These persons mainly run NDT tests. To have some confidence we're not breaking their workflows, let us periodically run a MK probe from macOS against the testing infra. This end-to-end test is not very precise, because we are not running the same version they are running and maybe it's not even the same OS and we likely have different dependencies. Yet, it seems a reasonable best effort at ensuring we ain't breaking things too badly. See ooni/backend#461 (comment) * fix: tweak legacy probe versions and targets Limit the runs to versions of the legacy probes that have been running non-ndt-measurements in the last seven days. See ooni/backend#461 (comment). Make sure we target both production and testing. As far as testing is concerned, just point to the HTTPS version to ensure that there are no API breakages for now. * fix: tweak probe-cli related checks 1. ensure we build the most relevant versions (unfortunately we still have some people who's running probe-cli 3.0.0 with MK) 2. ensure we test both prod and testing
ooni · Nov 13, 2020 · 906bf1e · 906bf1e
1 parent f6c458c
commit 906bf1e
Show file tree

Hide file tree

Showing 8 changed files with 79 additions and 14 deletions.
diff --git a/.github/workflows/debian9.yml b/.github/workflows/debian9.yml
@@ -4,6 +4,8 @@
 # As of 2020-09-24, this is a significant fraction of all OONI measurements.
 #
 # See also analysis at: https://github.com/ooni/backend/issues/446#issuecomment-697172389
+#
+# Status as of 2020-11-13: https://github.com/ooni/backend/issues/461#issuecomment-726587481.
 name: debian9
 on:
   pull_request:
@@ -17,10 +19,9 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        - "ooniprobe_2.0.2"
-        - "ooniprobe_2.1.0"
-        - "ooniprobe_2.2.0"
-        - "ooniprobe_2.3.0"
+        - "ooniprobe_2.1.0" # 47.933993% of measurements as of 2020-11-13
+        - "ooniprobe_2.2.0" # ~4% of measurements as of 2020-11-13
+        - "ooniprobe_2.3.0" # 2.470149 of measurements as of 2020-11-13
     steps:
       - uses: actions/setup-go@v1
         with:

diff --git a/.github/workflows/linuxmk0.10.11.yml b/.github/workflows/linuxmk0.10.11.yml
@@ -1,8 +1,11 @@
 # As of 2020-09-24 we have a significant user base running ooniprobe-cli
 # from Linux systems. The version they're running contains MK 0.10.11. So
-# here we're using the docker container that built sch version.
+# here we're using the docker container that built such version.
 #
 # See also analysis at https://github.com/ooni/backend/issues/446#issuecomment-697172389.
+#
+# We still see 1.399000% of measurements as of 2020-11-13 coming from
+# cli v3.0.0. which includes MK v0.10.11.
 name: libmk0.10.11
 on:
   pull_request:
@@ -12,6 +15,12 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        extraoptions:
+        - ""
+        - "--bouncer=https://ams-pg-test.ooni.org"
     steps:
       - uses: actions/setup-go@v1
         with:
@@ -24,6 +33,7 @@ jobs:
           done
           docker run -v`pwd`:/mk -w/mk openobservatory/mk-alpine:20200226 \
             measurement_kit -o output/mk.jsonl                            \
+                ${{ matrix.extraoptions }}                                \
                 --ca-bundle-path ca-bundle.pem                            \
                 --geoip-country-path country.mmdb                         \
                 --geoip-asn-path asn.mmdb                                 \

diff --git a/.github/workflows/macos10.15.yml b/.github/workflows/macos10.15.yml
@@ -2,6 +2,9 @@
 # ooniprobe 2.3.0 from macOS. So, here we just do that.
 #
 # See also https://github.com/ooni/backend/issues/446#issuecomment-697172389
+#
+# Status as of 2020-11-13: https://github.com/ooni/backend/issues/461#issuecomment-726587481
+# where ooniprobe 2.3.0 on macos is 17.641921% of the measurements.
 name: macos10.15
 on:
   pull_request:
@@ -16,17 +19,30 @@ jobs:
         with:
           go-version: "1.14"
       - uses: actions/checkout@v2
+      #
       # Limitation: we test the cloudfronted endpoint as a normal HTTPS endpoint.
+      #
       # Limitation: the original cloudfronted endpoint is no longer available, thus
-      # we are testing a new endpoint we setup for this purpose.
+      # we are testing a new endpoint we setup for this purpose. See the error
+      # available in vagrant/debian9/boostrap.
+      #
       - run: |
           brew install ooniprobe
           install -d $HOME/.ooni
           touch $HOME/.ooni/initialized
+
           ooniprobe -P https -o ./output/https.jsonl \
             web_connectivity -u http://mail.google.com
-          ooniprobe -P https --bouncer=https://dvp6h0xblpcqp.cloudfront.net -o ./output/https.jsonl \
+
+          ooniprobe -P https --bouncer=https://dvp6h0xblpcqp.cloudfront.net \
+            -o ./output/cloudfront.jsonl \
             web_connectivity -u http://mail.google.com
+
           ooniprobe -P onion -o ./output/onion.jsonl \
             web_connectivity -u http://mail.google.com
+          
+          ooniprobe -P https --bouncer=https://ams-pg-test.ooni.org \
+            -o ./output/https-test-infra.jsonl \
+            web_connectivity -u http://mail.google.com
+
       - run: go run ./script/postprocess.go
diff --git a/.github/workflows/measurementkit.yml b/.github/workflows/measurementkit.yml
@@ -1,5 +1,13 @@
 # Not very popular among users but yet useful to garner confidence that
 # MK is WAI, modulo possible dependencies issues.
+#
+# As of 2020-11-13, we don't have any significant OONI client running MK
+# but we know that there are many legacy clients running `ndt` using version
+# of MK compiled by people independently of OONI. So we are keeping this
+# workflow for now, to be sure we are not breaking this use case.
+#
+# Because this is increasingly less important to us, we only run tests
+# using OONI's testing infrastructure.
 name: measurementkit
 on:
   pull_request:
@@ -22,6 +30,7 @@ jobs:
             gunzip -f $f
           done
           measurement_kit -o output/mk.jsonl             \
+                --bouncer=https://ams-pg-test.ooni.org/  \
                 --ca-bundle-path ca-bundle.pem           \
                 --geoip-country-path country.mmdb        \
                 --geoip-asn-path asn.mmdb                \

diff --git a/.github/workflows/probeengine.yml b/.github/workflows/probeengine.yml
@@ -16,9 +16,11 @@ jobs:
       matrix:
         version:
         - "master"
+        - "v0.18.0"  # {ios,android} v2.7.0; desktop v3.0.4; cli v3.0.9 [2020-11-13]
         options:
         - "-o output/miniooni.jsonl"
         - "--probe-services=https://dvp6h0xblpcqp.cloudfront.net -o output/miniooni.jsonl"
+        - "--probe-services=https://ams-pg-test.ooni.org -o output/miniooni.jsonl"
     steps:
       - uses: actions/setup-go@v1
         with:

diff --git a/.github/workflows/probeengine1.yml b/.github/workflows/probeengine1.yml
@@ -14,10 +14,10 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        - "v0.15.1"
-        - "v0.13.2"
+        - "v0.16.0" # 2.243656% of measurements (cli v3.0.7-beta.1) [2020-11-13]
         options:
         - "-o output/miniooni.jsonl"
+        - "--probe-services-url=https://ams-pg-test.ooni.org/ -o output/miniooni.jsonl"
     steps:
       - uses: actions/setup-go@v1
         with:

diff --git a/.github/workflows/probeengine2.yml b/.github/workflows/probeengine2.yml
@@ -14,11 +14,10 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        - "7ac6a37960"  # v0.9.0+patches
-        - "4bcf3efabf"  # v0.10.0
-        - "v0.11.0"
+        - "v0.10.0" # 1.399000% of measurements as of 2020-11-13 (cli v3.0.0)
         options:
         - "-o output/miniooni.jsonl"
+        - "--bouncer=https://ams-pg-test.ooni.org/ -o output/miniooni.jsonl"
     steps:
       - uses: actions/setup-go@v1
         with:

diff --git a/vagrant/debian9/bootstrap b/vagrant/debian9/bootstrap
@@ -4,12 +4,40 @@ echo "$@" | tee -a /etc/apt/sources.list
 apt-get update
 apt-get install -y ooniprobe
 touch /var/lib/ooni/initialized
+
+#
 # Limitation: we test the cloudfronted endpoint as a normal HTTPS endpoint.
+#
 # Limitation: the original cloudfronted endpoint is no longer available, thus
-# we are testing a new endpoint we setup for this purpose.
+# we are testing a new endpoint we setup for this purpose:
+#
+# ```
+# % ooniprobe -P cloudfront http_invalid_request_line
+# [!] Error processing dns_spoof.py
+# [!] Error processing traceroute.py
+# Looking up your IP address via ubuntu
+# Found your IP via a GeoIP service
+# Looking up collector and test helpers with https://a0.awsstatic.com
+# Unhandled Error
+# Traceback (most recent call last):
+# Failure: ooni.errors.OONIBError:
+#
+# [!] Could not start the director
+# [!] Could not find a valid collector.
+# Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.
+# ```
+#
+
 ooniprobe -P https -o /vagrant/https.jsonl \
     web_connectivity -u http://mail.google.com
-ooniprobe -P https --bouncer=https://dvp6h0xblpcqp.cloudfront.net -o ./output/https.jsonl \
+
+ooniprobe -P https --bouncer=https://dvp6h0xblpcqp.cloudfront.net \
+    -o ./output/cloudfront.jsonl \
     web_connectivity -u http://mail.google.com
+
 ooniprobe -P onion -o /vagrant/onion.jsonl \
     web_connectivity -u http://mail.google.com
+
+ooniprobe -P https --bouncer=https://ams-pg-test.ooni.org \
+    -o ./output/https-test-infra.jsonl \
+    web_connectivity -u http://mail.google.com