feat(statsd): Measure number of project config attempts (#1281)

Track a metric for how often project configs are re-attempted, either
because they received a "pending" response or an error response.

relay-server/src/actors/project_upstream.rs

@@ -90,6 +90,7 @@ struct ProjectStateChannel {
     receiver: Shared<oneshot::Receiver<Arc<ProjectState>>>,
     deadline: Instant,
     no_cache: bool,
+    attempts: u64,
 }
 
 impl ProjectStateChannel {
@@ -101,6 +102,7 @@ impl ProjectStateChannel {
             receiver: receiver.shared(),
             deadline: Instant::now() + timeout,
             no_cache: false,
+            attempts: 0,
         }
     }
 
@@ -196,7 +198,10 @@ impl UpstreamProjectSource {
         let requests: Vec<_> = (cache_batches.into_iter())
             .chain(nocache_batches.into_iter())
             .map(|channels_batch| {
-                let channels_batch: BTreeMap<_, _> = channels_batch.collect();
+                let mut channels_batch: BTreeMap<_, _> = channels_batch.collect();
+                for channel in channels_batch.values_mut() {
+                    channel.attempts += 1;
+                }
                 relay_log::debug!("sending request of size {}", channels_batch.len());
                 metric!(
                     histogram(RelayHistograms::ProjectStateRequestBatchSize) =
@@ -261,7 +266,10 @@ impl UpstreamProjectSource {
                                         Some(ProjectState::err())
                                     })
                                     .unwrap_or_else(ProjectState::missing);
-
+                                metric!(
+                                    histogram(RelayHistograms::ProjectStateAttempts) =
+                                        channel.attempts
+                                );
                                 channel.send(state.sanitize());
                             }
                         }

relay-server/src/statsd.rs

@@ -89,6 +89,8 @@ pub enum RelayHistograms {
     ///
     /// See `project_cache.size` for more description of the project cache.
     ProjectStateReceived,
+    /// Number of attempts required to fetch the config for a given project key.
+    ProjectStateAttempts,
     /// Number of project states currently held in the in-memory project cache.
     ///
     /// The cache duration for project states can be configured with the following options:
@@ -150,6 +152,7 @@ impl HistogramMetric for RelayHistograms {
             RelayHistograms::RequestSizeBytesRaw => "event.size_bytes.raw",
             RelayHistograms::RequestSizeBytesUncompressed => "event.size_bytes.uncompressed",
             RelayHistograms::ProjectStatePending => "project_state.pending",
+            RelayHistograms::ProjectStateAttempts => "project_state.attempts",
             RelayHistograms::ProjectStateRequestBatchSize => "project_state.request.batch_size",
             RelayHistograms::ProjectStateReceived => "project_state.received",
             RelayHistograms::ProjectStateCacheSize => "project_cache.size",