Skip to content

Commit

Permalink
cleanup(minipipeline): zap DNSPossiblyNonexistingDomains metric (#1416)
Browse files Browse the repository at this point in the history
What we can do with DNSPossiblyNonexistingDomains, we can also do with
DNSLookupExpectedFailure.

Part of ooni/probe#2634
  • Loading branch information
bassosimone authored Nov 30, 2023
1 parent 4f4798e commit 51bcd6e
Show file tree
Hide file tree
Showing 69 changed files with 68 additions and 209 deletions.
3 changes: 1 addition & 2 deletions internal/cmd/minipipeline/testdata/analysis.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"x-drupal-cache": true,
"x-generator": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
3 changes: 1 addition & 2 deletions internal/cmd/minipipeline/testdata/analysis_classic.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"x-drupal-cache": true,
"x-generator": true
},
"DNSPossiblyNonexistingDomains": null
}
}
69 changes: 0 additions & 69 deletions internal/minipipeline/analysis.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package minipipeline

import (
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/optional"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
Expand All @@ -19,7 +18,6 @@ func AnalyzeWebObservations(container *WebObservationsContainer) *WebAnalysis {
analysis.httpComputeFailureMetrics(container)
analysis.httpComputeFinalResponseMetrics(container)

analysis.ComputeDNSPossiblyNonexistingDomains(container)
return analysis
}

Expand Down Expand Up @@ -129,12 +127,6 @@ type WebAnalysis struct {

// HTTPFinalResponseDiffUncommonHeadersIntersection contains the uncommon headers intersection.
HTTPFinalResponseDiffUncommonHeadersIntersection optional.Value[map[string]bool]

// TODO(bassosimone): there are probably redundant metrics from this point on

// DNSPossiblyNonexistingDomains lists all the domains for which both
// the probe and the TH failed to perform DNS lookups.
DNSPossiblyNonexistingDomains optional.Value[map[string]bool]
}

func (wa *WebAnalysis) dnsComputeSuccessMetrics(c *WebObservationsContainer) {
Expand Down Expand Up @@ -576,64 +568,3 @@ func (wa *WebAnalysis) httpDiffTitleDifferentLongWords(obs *WebObservation) {

wa.HTTPFinalResponseDiffTitleDifferentLongWords = optional.Some(state)
}

// ComputeDNSPossiblyNonexistingDomains computes the DNSPossiblyNonexistingDomains field.
func (wa *WebAnalysis) ComputeDNSPossiblyNonexistingDomains(c *WebObservationsContainer) {
var state map[string]bool

// first inspect the failures
for _, obs := range c.DNSLookupFailures {
// skip the comparison if we don't have enough information
if obs.DNSLookupFailure.IsNone() || obs.ControlDNSLookupFailure.IsNone() {
continue
}

// flip state from None to empty when we see the first couple of
// (probe, th) failures allowing us to perform a comparison
if state == nil {
state = make(map[string]bool)
}

// assume the domain is set in both cases
domain := obs.DNSDomain.Unwrap()
runtimex.Assert(domain == obs.ControlDNSDomain.Unwrap(), "mismatch between domain names")

// a domain is nonexisting if both the probe and the TH say so
if obs.DNSLookupFailure.Unwrap() != netxlite.FailureDNSNXDOMAINError {
continue
}
if obs.ControlDNSLookupFailure.Unwrap() != "dns_name_error" {
continue
}

// set the state
state[domain] = true
}

// then inspect the successes
for _, obs := range c.DNSLookupSuccesses {
// skip the comparison if we don't have enough information
if obs.DNSLookupFailure.IsNone() && obs.ControlDNSLookupFailure.IsNone() {
continue
}

// assume the domain is always set
domain := obs.DNSDomain.Unwrap()

// clear the state if the probe succeeded
if !obs.DNSLookupFailure.IsNone() && obs.DNSLookupFailure.Unwrap() == "" {
delete(state, domain)
continue
}

// clear the state if the TH succeded
if !obs.ControlDNSLookupFailure.IsNone() && obs.ControlDNSLookupFailure.Unwrap() == "" {
runtimex.Assert(domain == obs.ControlDNSDomain.Unwrap(), "mismatch between domain names")
delete(state, domain)
continue
}
}

// note that optional.Some constructs None if state is nil
wa.DNSPossiblyNonexistingDomains = optional.Some(state)
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"alt-svc": true,
"content-length": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"alt-svc": true,
"content-length": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"alt-svc": true,
"content-length": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"alt-svc": true,
"content-length": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"alt-svc": true,
"content-length": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,5 @@
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"alt-svc": true,
"content-length": true
},
"DNSPossiblyNonexistingDomains": null
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,5 @@
"HTTPFinalResponseDiffTitleDifferentLongWords": {},
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"content-length": true
},
"DNSPossiblyNonexistingDomains": {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,5 @@
"HTTPFinalResponseDiffTitleDifferentLongWords": {},
"HTTPFinalResponseDiffUncommonHeadersIntersection": {
"content-length": true
},
"DNSPossiblyNonexistingDomains": null
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,5 @@
"default": true,
"denied": true
},
"HTTPFinalResponseDiffUncommonHeadersIntersection": {},
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,5 @@
"default": true,
"denied": true
},
"HTTPFinalResponseDiffUncommonHeadersIntersection": {},
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,5 @@
"default": true,
"denied": true
},
"HTTPFinalResponseDiffUncommonHeadersIntersection": {},
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,5 @@
"default": true,
"denied": true
},
"HTTPFinalResponseDiffUncommonHeadersIntersection": {},
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": {}
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,5 @@
"HTTPFinalResponseDiffBodyProportionFactor": null,
"HTTPFinalResponseDiffStatusCodeMatch": null,
"HTTPFinalResponseDiffTitleDifferentLongWords": null,
"HTTPFinalResponseDiffUncommonHeadersIntersection": null,
"DNSPossiblyNonexistingDomains": null
"HTTPFinalResponseDiffUncommonHeadersIntersection": null
}
Loading

0 comments on commit 51bcd6e

Please sign in to comment.