Skip to content

Commit

Permalink
add histogram to more core s3 calls
Browse files Browse the repository at this point in the history
and make their histograms appear only when the endpoint has been called
  • Loading branch information
trinity-1686a committed Nov 21, 2024
1 parent 70a248b commit 125c509
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 6 deletions.
36 changes: 30 additions & 6 deletions quickwit/quickwit-storage/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,13 @@ pub struct StorageMetrics {

pub object_storage_delete_requests_total: IntCounter,
pub object_storage_bulk_delete_requests_total: IntCounter,
pub object_storage_delete_request_duration: Histogram,
pub object_storage_bulk_delete_request_duration: Histogram,
pub object_storage_delete_request_duration: Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
pub object_storage_bulk_delete_request_duration:
Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
pub object_storage_get_request_duration: Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
pub object_storage_put_request_duration: Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
pub object_storage_put_part_request_duration:
Lazy<Histogram, Box<dyn Fn() -> Histogram + Send>>,
}

impl Default for StorageMetrics {
Expand Down Expand Up @@ -86,10 +91,26 @@ impl Default for StorageMetrics {
["action"],
vec![0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0],
);
let object_storage_delete_request_duration =
object_storage_request_duration.with_label_values(["delete_object"]);
let object_storage_bulk_delete_request_duration =
object_storage_request_duration.with_label_values(["delete_objects"]);
let object_storage_request_duration_clone = object_storage_request_duration.clone();
let object_storage_delete_request_duration = Lazy::new(Box::new(move || {
object_storage_request_duration_clone.with_label_values(["delete_object"])
}) as _);
let object_storage_request_duration_clone = object_storage_request_duration.clone();
let object_storage_bulk_delete_request_duration = Lazy::new(Box::new(move || {
object_storage_request_duration_clone.with_label_values(["delete_objects"])
}) as _);
let object_storage_request_duration_clone = object_storage_request_duration.clone();
let object_storage_get_request_duration = Lazy::new(Box::new(move || {
object_storage_request_duration_clone.with_label_values(["get"])
}) as _);
let object_storage_request_duration_clone = object_storage_request_duration.clone();
let object_storage_put_request_duration = Lazy::new(Box::new(move || {
object_storage_request_duration_clone.with_label_values(["put"])
}) as _);
let object_storage_request_duration_clone = object_storage_request_duration.clone();
let object_storage_put_part_request_duration = Lazy::new(Box::new(move || {
object_storage_request_duration_clone.with_label_values(["put_part"])
}) as _);

StorageMetrics {
fast_field_cache: CacheMetrics::for_component("fastfields"),
Expand Down Expand Up @@ -142,6 +163,9 @@ impl Default for StorageMetrics {
object_storage_bulk_delete_requests_total,
object_storage_delete_request_duration,
object_storage_bulk_delete_request_duration,
object_storage_get_request_duration,
object_storage_put_request_duration,
object_storage_put_part_request_duration,
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,9 @@ impl S3CompatibleObjectStorage {
crate::STORAGE_METRICS
.object_storage_upload_num_bytes
.inc_by(len);
let _timer = crate::STORAGE_METRICS
.object_storage_put_request_duration
.start_timer();

self.s3_client
.put_object()
Expand Down Expand Up @@ -429,6 +432,9 @@ impl S3CompatibleObjectStorage {
crate::STORAGE_METRICS
.object_storage_upload_num_bytes
.inc_by(part.len());
let _timer = crate::STORAGE_METRICS
.object_storage_put_part_request_duration
.start_timer();

let upload_part_output = self
.s3_client
Expand Down Expand Up @@ -549,6 +555,9 @@ impl S3CompatibleObjectStorage {
let range_str = range_opt.map(|range| format!("bytes={}-{}", range.start, range.end - 1));

crate::STORAGE_METRICS.object_storage_get_total.inc();
let _timer = crate::STORAGE_METRICS
.object_storage_get_request_duration
.start_timer();

let get_object_output = self
.s3_client
Expand Down

0 comments on commit 125c509

Please sign in to comment.